BorisTM commited on
Commit
8ae7fd7
·
0 Parent(s):

Clean StaRSE-512 repository state

Browse files
Files changed (38) hide show
  1. .gitattributes +3 -0
  2. 1_Normalize/config.json +1 -0
  3. LICENSE.txt +202 -0
  4. README.md +85 -0
  5. assets/rumteb_cpu_latency.png +3 -0
  6. binary_static_embedding.py +223 -0
  7. binary_static_embedding_config.json +6 -0
  8. config_sentence_transformers.json +9 -0
  9. model.safetensors +3 -0
  10. modules.json +14 -0
  11. results/mteb_rus_v1_1/mteb_eval_results.json +0 -0
  12. results/mteb_rus_v1_1/mteb_eval_summary.json +341 -0
  13. results/mteb_rus_v1_1/raw_task_results/000_GeoreviewClassification.json +142 -0
  14. results/mteb_rus_v1_1/raw_task_results/001_HeadlineClassification.json +142 -0
  15. results/mteb_rus_v1_1/raw_task_results/002_InappropriatenessClassification.json +142 -0
  16. results/mteb_rus_v1_1/raw_task_results/003_KinopoiskClassification.json +142 -0
  17. results/mteb_rus_v1_1/raw_task_results/004_MassiveIntentClassification.json +272 -0
  18. results/mteb_rus_v1_1/raw_task_results/005_MassiveScenarioClassification.json +272 -0
  19. results/mteb_rus_v1_1/raw_task_results/006_RuReviewsClassification.json +142 -0
  20. results/mteb_rus_v1_1/raw_task_results/007_RuSciBenchGRNTIClassification.json +142 -0
  21. results/mteb_rus_v1_1/raw_task_results/008_RuSciBenchOECDClassification.json +142 -0
  22. results/mteb_rus_v1_1/raw_task_results/009_GeoreviewClusteringP2P.json +37 -0
  23. results/mteb_rus_v1_1/raw_task_results/010_RuSciBenchGRNTIClusteringP2P.json +37 -0
  24. results/mteb_rus_v1_1/raw_task_results/011_RuSciBenchOECDClusteringP2P.json +37 -0
  25. results/mteb_rus_v1_1/raw_task_results/012_CEDRClassification.json +87 -0
  26. results/mteb_rus_v1_1/raw_task_results/013_SensitiveTopicsClassification.json +87 -0
  27. results/mteb_rus_v1_1/raw_task_results/014_TERRa.json +51 -0
  28. results/mteb_rus_v1_1/raw_task_results/015_MIRACLReranking.json +169 -0
  29. results/mteb_rus_v1_1/raw_task_results/016_RuBQReranking.json +169 -0
  30. results/mteb_rus_v1_1/raw_task_results/017_MIRACLRetrievalHardNegatives_v2.json +169 -0
  31. results/mteb_rus_v1_1/raw_task_results/018_RiaNewsRetrievalHardNegatives_v2.json +169 -0
  32. results/mteb_rus_v1_1/raw_task_results/019_RuBQRetrieval.json +169 -0
  33. results/mteb_rus_v1_1/raw_task_results/020_RUParaPhraserSTS.json +29 -0
  34. results/mteb_rus_v1_1/raw_task_results/021_STS22.json +29 -0
  35. results/mteb_rus_v1_1/raw_task_results/022_RuSTSBenchmarkSTS.json +29 -0
  36. results/mteb_rus_v1_1/summary_compact.json +53 -0
  37. starse_static_meta.json +20 -0
  38. tokenizer.json +0 -0
.gitattributes ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ *.bin filter=lfs diff=lfs merge=lfs -text
2
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
3
+ assets/rumteb_cpu_latency.png filter=lfs diff=lfs merge=lfs -text
1_Normalize/config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {}
LICENSE.txt ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
README.md ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - ru
4
+ library_name: sentence-transformers
5
+ pipeline_tag: sentence-similarity
6
+ license: apache-2.0
7
+ tags:
8
+ - sentence-transformers
9
+ - sentence-similarity
10
+ - feature-extraction
11
+ - static-embeddings
12
+ - binary
13
+ - russian
14
+ ---
15
+
16
+ # StaRSE-512
17
+
18
+ **StaRSE** stands for **Sta**tic **R**ussian **S**entence **E**mbeddings. It is a compact Russian sentence embedding model implemented as a
19
+ [Sentence-Transformers](https://www.sbert.net/) `StaticEmbedding`
20
+ endpoint.
21
+
22
+ The model is intended for CPU-friendly semantic similarity, clustering,
23
+ classification features, and retrieval-style first-stage representations when a
24
+ full Transformer encoder is too expensive to run at high throughput.
25
+
26
+ ![RuMTEB quality-latency trade-off](assets/rumteb_cpu_latency.png)
27
+
28
+ ## Performance
29
+
30
+ Evaluation is reported on
31
+ [`MTEB(rus, v1.1)`](https://docs.mteb.org/overview/available_benchmarks/#mtebrus-v11)
32
+ across 23 tasks. The main score is `mean_task_main_score = 51.16`.
33
+
34
+ | Task type | Tasks | Mean score |
35
+ |---|---:|---:|
36
+ | Classification | 9 | 56.81 |
37
+ | Clustering | 3 | 51.80 |
38
+ | MultilabelClassification | 2 | 35.01 |
39
+ | PairClassification | 1 | 52.50 |
40
+ | Reranking | 2 | 41.88 |
41
+ | Retrieval | 3 | 39.09 |
42
+ | STS | 3 | 62.18 |
43
+
44
+ ## Usage
45
+
46
+ Install [Sentence Transformers](https://www.sbert.net/docs/installation.html):
47
+
48
+ ```bash
49
+ pip install -U sentence-transformers
50
+ ```
51
+
52
+ Load the model with `trust_remote_code=True`.
53
+
54
+ ```python
55
+ from sentence_transformers import SentenceTransformer
56
+
57
+ model = SentenceTransformer("BorisTM/starse-512", trust_remote_code=True)
58
+
59
+ sentences = [
60
+ "Партитуры Чайковского часто звучат в консерватории.",
61
+ "Балетная сцена хранит музыку Щелкунчика.",
62
+ "Футбольная команда выиграла матч.",
63
+ ]
64
+
65
+ embeddings = model.encode(sentences, normalize_embeddings=True)
66
+ similarities = model.similarity(embeddings, embeddings)
67
+ print(embeddings.shape) # (3, 512)
68
+ print(tuple(similarities.shape)) # (3, 3)
69
+ print(similarities)
70
+ # tensor([[1.0000, 0.3521, 0.0626],
71
+ # [0.3521, 1.0000, 0.0420],
72
+ # [0.0626, 0.0420, 1.0000]])
73
+ ```
74
+
75
+
76
+ ## Citation
77
+
78
+ ```bibtex
79
+ @misc{starse2026,
80
+ title = {TBD},
81
+ author = {TBD},
82
+ year = {TBD},
83
+ url = {https://huggingface.co/BorisTM/starse-512}
84
+ }
85
+ ```
assets/rumteb_cpu_latency.png ADDED

Git LFS Details

  • SHA256: 4c6bf647300fc5fb62d4cfdd035736cc609349a43d2338a421fb5aa7b6889e34
  • Pointer size: 131 Bytes
  • Size of remote file: 154 kB
binary_static_embedding.py ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sign-packed StaticEmbedding for sentence-transformers.
2
+
3
+ Compact storage: every embedding row is represented as a sign bitmask (one bit
4
+ per dimension, packed into uint8 bytes) plus a per-row L2 norm. At load time the
5
+ module reconstructs a float ``EmbeddingBag`` lookup table identical to what a
6
+ trained ``norm * sign(unit) / sqrt(dim)`` projection would produce, so inference
7
+ behaves like a regular :class:`StaticEmbedding`.
8
+
9
+ On disk the model is ~30x smaller than the fp32 form. To use it via
10
+ ``SentenceTransformer``, pass ``trust_remote_code=True``::
11
+
12
+ from sentence_transformers import SentenceTransformer
13
+ model = SentenceTransformer("BorisTM/starse-512", trust_remote_code=True)
14
+ embeddings = model.encode(["пример"])
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import math
20
+ import os
21
+ from pathlib import Path
22
+ from typing import Any
23
+
24
+ try:
25
+ from typing import Self
26
+ except ImportError:
27
+ from typing_extensions import Self
28
+
29
+ import numpy as np
30
+ import torch
31
+ from safetensors.torch import load_file as load_safetensors_file
32
+ from safetensors.torch import save_file as save_safetensors_file
33
+ from tokenizers import Tokenizer
34
+ from torch import nn
35
+ from transformers import PreTrainedTokenizerFast
36
+
37
+ from sentence_transformers.base.modules.input_module import InputModule
38
+
39
+
40
+ class BinaryStaticEmbedding(InputModule):
41
+ """1-bit sign + per-row L2 norm StaticEmbedding."""
42
+
43
+ modalities: list[str] = ["text"]
44
+ config_keys: list[str] = ["embedding_dim", "vocab_size"]
45
+ config_file_name: str = "binary_static_embedding_config.json"
46
+ weights_file_name: str = "model.safetensors"
47
+ tokenizer_file_name: str = "tokenizer.json"
48
+
49
+ def __init__(
50
+ self,
51
+ tokenizer: Tokenizer | PreTrainedTokenizerFast,
52
+ embedding_dim: int,
53
+ vocab_size: int,
54
+ packed_signs: torch.Tensor | np.ndarray | None = None,
55
+ norms: torch.Tensor | np.ndarray | None = None,
56
+ embedding_weights: torch.Tensor | np.ndarray | None = None,
57
+ **kwargs,
58
+ ) -> None:
59
+ super().__init__()
60
+
61
+ if isinstance(tokenizer, PreTrainedTokenizerFast):
62
+ tokenizer = tokenizer._tokenizer
63
+ elif not isinstance(tokenizer, Tokenizer):
64
+ raise ValueError("tokenizer must be a fast tokenizer (Tokenizer or PreTrainedTokenizerFast)")
65
+ self.tokenizer: Tokenizer = tokenizer
66
+ self.tokenizer.no_padding()
67
+
68
+ self.embedding_dim = int(embedding_dim)
69
+ self.vocab_size = int(vocab_size)
70
+
71
+ if embedding_weights is not None:
72
+ weight_tensor = _as_float_tensor(embedding_weights)
73
+ if weight_tensor.shape != (self.vocab_size, self.embedding_dim):
74
+ raise ValueError(
75
+ f"embedding_weights shape {tuple(weight_tensor.shape)} does not match "
76
+ f"(vocab_size={self.vocab_size}, embedding_dim={self.embedding_dim})"
77
+ )
78
+ elif packed_signs is not None and norms is not None:
79
+ weight_tensor = self._unpack_to_lookup(
80
+ packed_signs=_as_uint8_tensor(packed_signs),
81
+ norms=_as_float_tensor(norms),
82
+ embedding_dim=self.embedding_dim,
83
+ )
84
+ else:
85
+ weight_tensor = torch.zeros((self.vocab_size, self.embedding_dim), dtype=torch.float32)
86
+
87
+ self.embedding = nn.EmbeddingBag.from_pretrained(weight_tensor, freeze=True)
88
+ self.num_embeddings = self.embedding.num_embeddings
89
+ # For the model card
90
+ self.base_model = kwargs.get("base_model", None)
91
+
92
+ # ------------------------------------------------------------------ utils
93
+ @staticmethod
94
+ def _unpack_to_lookup(packed_signs: torch.Tensor, norms: torch.Tensor, embedding_dim: int) -> torch.Tensor:
95
+ """Reconstruct a float ``[vocab, dim]`` lookup from packed sign bits and per-row norms."""
96
+ if packed_signs.dtype != torch.uint8:
97
+ raise TypeError(f"packed_signs must be uint8, got {packed_signs.dtype}")
98
+ expected_packed_dim = (embedding_dim + 7) // 8
99
+ if packed_signs.dim() != 2 or packed_signs.shape[1] != expected_packed_dim:
100
+ raise ValueError(
101
+ f"packed_signs shape {tuple(packed_signs.shape)} does not match (vocab, ceil(dim/8)={expected_packed_dim})"
102
+ )
103
+ bits = np.unpackbits(packed_signs.cpu().numpy(), axis=1, bitorder="big")[:, :embedding_dim]
104
+ signs = bits.astype(np.float32) * 2.0 - 1.0 # 0 -> -1, 1 -> +1
105
+ scale = norms.detach().to(torch.float32).cpu().unsqueeze(1) / math.sqrt(embedding_dim)
106
+ return (torch.from_numpy(signs) * scale).contiguous()
107
+
108
+ @staticmethod
109
+ def _pack_from_weight(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
110
+ """Inverse of ``_unpack_to_lookup``: extract packed sign bits and per-row norms."""
111
+ weight = weight.detach().float().cpu()
112
+ norms = torch.linalg.vector_norm(weight, dim=1).clamp_min(1e-12)
113
+ signs = (weight >= 0).to(torch.uint8).numpy()
114
+ packed = np.packbits(signs, axis=1, bitorder="big")
115
+ return torch.from_numpy(packed), norms
116
+
117
+ # ------------------------------------------------------------------ forward
118
+ def preprocess(self, inputs: list[str], prompt: str | None = None, **kwargs) -> dict[str, torch.Tensor]:
119
+ if prompt:
120
+ inputs = self._prepend_prompt(inputs, prompt)
121
+ encodings = self.tokenizer.encode_batch(inputs, add_special_tokens=False)
122
+ encodings_ids = [encoding.ids for encoding in encodings]
123
+ offsets = torch.from_numpy(
124
+ np.cumsum([0] + [len(token_ids) for token_ids in encodings_ids[:-1]])
125
+ )
126
+ input_ids = torch.tensor(
127
+ [token_id for token_ids in encodings_ids for token_id in token_ids],
128
+ dtype=torch.long,
129
+ )
130
+ return {"input_ids": input_ids, "offsets": offsets}
131
+
132
+ def forward(self, features: dict[str, torch.Tensor], **kwargs) -> dict[str, torch.Tensor]:
133
+ features["sentence_embedding"] = self.embedding(features["input_ids"], features["offsets"])
134
+ return features
135
+
136
+ @property
137
+ def max_seq_length(self) -> int:
138
+ return math.inf
139
+
140
+ def get_embedding_dimension(self) -> int:
141
+ return self.embedding_dim
142
+
143
+ # ------------------------------------------------------------------ persistence
144
+ def save(self, output_path: str, *args, safe_serialization: bool = True, **kwargs) -> None:
145
+ output_path = Path(output_path)
146
+ output_path.mkdir(parents=True, exist_ok=True)
147
+ packed_signs, norms = self._pack_from_weight(self.embedding.weight)
148
+ save_safetensors_file(
149
+ {"packed_signs": packed_signs, "norms": norms},
150
+ str(output_path / self.weights_file_name),
151
+ )
152
+ self.save_config(str(output_path))
153
+ self.tokenizer.save(str(output_path / self.tokenizer_file_name))
154
+
155
+ def save_config(self, output_path: str) -> None:
156
+ import json
157
+
158
+ payload = {
159
+ "embedding_dim": self.embedding_dim,
160
+ "vocab_size": self.vocab_size,
161
+ "packed_bit_order": "big",
162
+ "scale": "norm / sqrt(embedding_dim)",
163
+ }
164
+ with open(Path(output_path) / self.config_file_name, "w", encoding="utf-8") as handle:
165
+ json.dump(payload, handle, ensure_ascii=False, indent=2)
166
+
167
+ @classmethod
168
+ def load(
169
+ cls,
170
+ model_name_or_path: str,
171
+ subfolder: str = "",
172
+ token: bool | str | None = None,
173
+ cache_folder: str | None = None,
174
+ revision: str | None = None,
175
+ local_files_only: bool = False,
176
+ **kwargs,
177
+ ) -> Self:
178
+ hub_kwargs = {
179
+ "subfolder": subfolder,
180
+ "token": token,
181
+ "cache_folder": cache_folder,
182
+ "revision": revision,
183
+ "local_files_only": local_files_only,
184
+ }
185
+ config_path = cls.load_file_path(model_name_or_path, filename=cls.config_file_name, **hub_kwargs)
186
+ if config_path is None:
187
+ raise FileNotFoundError(f"{cls.config_file_name} not found at {model_name_or_path}")
188
+ import json
189
+
190
+ with open(config_path, "r", encoding="utf-8") as handle:
191
+ config = json.load(handle)
192
+
193
+ tokenizer_path = cls.load_file_path(model_name_or_path, filename=cls.tokenizer_file_name, **hub_kwargs)
194
+ tokenizer = Tokenizer.from_file(tokenizer_path)
195
+
196
+ weights_path = cls.load_file_path(model_name_or_path, filename=cls.weights_file_name, **hub_kwargs)
197
+ if weights_path is None:
198
+ raise FileNotFoundError(f"{cls.weights_file_name} not found at {model_name_or_path}")
199
+ state = load_safetensors_file(weights_path)
200
+ packed_signs = state["packed_signs"]
201
+ norms = state["norms"]
202
+
203
+ return cls(
204
+ tokenizer=tokenizer,
205
+ embedding_dim=int(config["embedding_dim"]),
206
+ vocab_size=int(config["vocab_size"]),
207
+ packed_signs=packed_signs,
208
+ norms=norms,
209
+ )
210
+
211
+
212
+ def _as_float_tensor(value: torch.Tensor | np.ndarray) -> torch.Tensor:
213
+ if isinstance(value, np.ndarray):
214
+ value = torch.from_numpy(value)
215
+ return value.detach().to(torch.float32)
216
+
217
+
218
+ def _as_uint8_tensor(value: torch.Tensor | np.ndarray) -> torch.Tensor:
219
+ if isinstance(value, np.ndarray):
220
+ value = torch.from_numpy(value)
221
+ if value.dtype != torch.uint8:
222
+ value = value.to(torch.uint8)
223
+ return value
binary_static_embedding_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "embedding_dim": 512,
3
+ "vocab_size": 120138,
4
+ "packed_bit_order": "big",
5
+ "scale": "norm / sqrt(embedding_dim)"
6
+ }
config_sentence_transformers.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "SentenceTransformer",
3
+ "similarity_fn_name": "cosine",
4
+ "default_prompt_name": null,
5
+ "prompts": {
6
+ "document": "",
7
+ "query": ""
8
+ }
9
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66d9dee585e3aa835e9bbb36f002d79e33a27a4567db708c80eae91fd60d76fd
3
+ size 8169544
modules.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "binary_static_embedding.BinaryStaticEmbedding"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Normalize",
12
+ "type": "sentence_transformers.models.Normalize"
13
+ }
14
+ ]
results/mteb_rus_v1_1/mteb_eval_results.json ADDED
The diff for this file is too large to render. See raw diff
 
results/mteb_rus_v1_1/mteb_eval_summary.json ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "ok",
3
+ "step": 0,
4
+ "benchmark": "MTEB(rus, v1.1)",
5
+ "task_count": 23,
6
+ "completed_count": 23,
7
+ "failed_count": 0,
8
+ "metric": "mean_task_main_score",
9
+ "mean_main_score": 0.5116301998770688,
10
+ "scores": {
11
+ "GeoreviewClassification": 0.370068359375,
12
+ "HeadlineClassification": 0.74287109375,
13
+ "InappropriatenessClassification": 0.591455078125,
14
+ "KinopoiskClassification": 0.5024,
15
+ "MassiveIntentClassification": 0.5962340282447882,
16
+ "MassiveScenarioClassification": 0.6775386684599866,
17
+ "RuReviewsClassification": 0.547607421875,
18
+ "RuSciBenchGRNTIClassification": 0.60283203125,
19
+ "RuSciBenchOECDClassification": 0.481982421875,
20
+ "GeoreviewClusteringP2P": 0.5399503935938068,
21
+ "RuSciBenchGRNTIClusteringP2P": 0.5394743194629961,
22
+ "RuSciBenchOECDClusteringP2P": 0.47453223039062903,
23
+ "CEDRClassification": 0.4139213602550478,
24
+ "SensitiveTopicsClassification": 0.286181640625,
25
+ "TERRa": 0.5250119340487216,
26
+ "MIRACLReranking": 0.29416,
27
+ "RuBQReranking": 0.54341,
28
+ "MIRACLRetrievalHardNegatives.v2": 0.23447,
29
+ "RiaNewsRetrievalHardNegatives.v2": 0.51899,
30
+ "RuBQRetrieval": 0.41913,
31
+ "RUParaPhraserSTS": 0.56304081451451,
32
+ "STS22": 0.588935863512821,
33
+ "RuSTSBenchmarkSTS": 0.7132969378142737
34
+ },
35
+ "records": [
36
+ {
37
+ "task": "GeoreviewClassification",
38
+ "metric": "main_score",
39
+ "score": 0.370068359375
40
+ },
41
+ {
42
+ "task": "HeadlineClassification",
43
+ "metric": "main_score",
44
+ "score": 0.74287109375
45
+ },
46
+ {
47
+ "task": "InappropriatenessClassification",
48
+ "metric": "main_score",
49
+ "score": 0.591455078125
50
+ },
51
+ {
52
+ "task": "KinopoiskClassification",
53
+ "metric": "main_score",
54
+ "score": 0.5024
55
+ },
56
+ {
57
+ "task": "MassiveIntentClassification",
58
+ "metric": "main_score",
59
+ "score": 0.5962340282447882
60
+ },
61
+ {
62
+ "task": "MassiveScenarioClassification",
63
+ "metric": "main_score",
64
+ "score": 0.6775386684599866
65
+ },
66
+ {
67
+ "task": "RuReviewsClassification",
68
+ "metric": "main_score",
69
+ "score": 0.547607421875
70
+ },
71
+ {
72
+ "task": "RuSciBenchGRNTIClassification",
73
+ "metric": "main_score",
74
+ "score": 0.60283203125
75
+ },
76
+ {
77
+ "task": "RuSciBenchOECDClassification",
78
+ "metric": "main_score",
79
+ "score": 0.481982421875
80
+ },
81
+ {
82
+ "task": "GeoreviewClusteringP2P",
83
+ "metric": "main_score",
84
+ "score": 0.5399503935938068
85
+ },
86
+ {
87
+ "task": "RuSciBenchGRNTIClusteringP2P",
88
+ "metric": "main_score",
89
+ "score": 0.5394743194629961
90
+ },
91
+ {
92
+ "task": "RuSciBenchOECDClusteringP2P",
93
+ "metric": "main_score",
94
+ "score": 0.47453223039062903
95
+ },
96
+ {
97
+ "task": "CEDRClassification",
98
+ "metric": "main_score",
99
+ "score": 0.4139213602550478
100
+ },
101
+ {
102
+ "task": "SensitiveTopicsClassification",
103
+ "metric": "main_score",
104
+ "score": 0.286181640625
105
+ },
106
+ {
107
+ "task": "TERRa",
108
+ "metric": "main_score",
109
+ "score": 0.5250119340487216
110
+ },
111
+ {
112
+ "task": "MIRACLReranking",
113
+ "metric": "main_score",
114
+ "score": 0.29416
115
+ },
116
+ {
117
+ "task": "RuBQReranking",
118
+ "metric": "main_score",
119
+ "score": 0.54341
120
+ },
121
+ {
122
+ "task": "MIRACLRetrievalHardNegatives.v2",
123
+ "metric": "main_score",
124
+ "score": 0.23447
125
+ },
126
+ {
127
+ "task": "RiaNewsRetrievalHardNegatives.v2",
128
+ "metric": "main_score",
129
+ "score": 0.51899
130
+ },
131
+ {
132
+ "task": "RuBQRetrieval",
133
+ "metric": "main_score",
134
+ "score": 0.41913
135
+ },
136
+ {
137
+ "task": "RUParaPhraserSTS",
138
+ "metric": "main_score",
139
+ "score": 0.56304081451451
140
+ },
141
+ {
142
+ "task": "STS22",
143
+ "metric": "main_score",
144
+ "score": 0.588935863512821
145
+ },
146
+ {
147
+ "task": "RuSTSBenchmarkSTS",
148
+ "metric": "main_score",
149
+ "score": 0.7132969378142737
150
+ }
151
+ ],
152
+ "task_timings": [
153
+ {
154
+ "task_index": 0,
155
+ "task_name": "GeoreviewClassification",
156
+ "task_type": "Classification",
157
+ "status": "completed",
158
+ "wall_seconds": 24.507260584970936,
159
+ "error": null
160
+ },
161
+ {
162
+ "task_index": 1,
163
+ "task_name": "HeadlineClassification",
164
+ "task_type": "Classification",
165
+ "status": "completed",
166
+ "wall_seconds": 27.876973578939214,
167
+ "error": null
168
+ },
169
+ {
170
+ "task_index": 2,
171
+ "task_name": "InappropriatenessClassification",
172
+ "task_type": "Classification",
173
+ "status": "completed",
174
+ "wall_seconds": 32.511821364983916,
175
+ "error": null
176
+ },
177
+ {
178
+ "task_index": 3,
179
+ "task_name": "KinopoiskClassification",
180
+ "task_type": "Classification",
181
+ "status": "completed",
182
+ "wall_seconds": 17.885735800024122,
183
+ "error": null
184
+ },
185
+ {
186
+ "task_index": 4,
187
+ "task_name": "MassiveIntentClassification",
188
+ "task_type": "Classification",
189
+ "status": "completed",
190
+ "wall_seconds": 61.33183418796398,
191
+ "error": null
192
+ },
193
+ {
194
+ "task_index": 5,
195
+ "task_name": "MassiveScenarioClassification",
196
+ "task_type": "Classification",
197
+ "status": "completed",
198
+ "wall_seconds": 30.563477964140475,
199
+ "error": null
200
+ },
201
+ {
202
+ "task_index": 6,
203
+ "task_name": "RuReviewsClassification",
204
+ "task_type": "Classification",
205
+ "status": "completed",
206
+ "wall_seconds": 32.07169775106013,
207
+ "error": null
208
+ },
209
+ {
210
+ "task_index": 7,
211
+ "task_name": "RuSciBenchGRNTIClassification",
212
+ "task_type": "Classification",
213
+ "status": "completed",
214
+ "wall_seconds": 42.64247382991016,
215
+ "error": null
216
+ },
217
+ {
218
+ "task_index": 8,
219
+ "task_name": "RuSciBenchOECDClassification",
220
+ "task_type": "Classification",
221
+ "status": "completed",
222
+ "wall_seconds": 41.84065520600416,
223
+ "error": null
224
+ },
225
+ {
226
+ "task_index": 9,
227
+ "task_name": "GeoreviewClusteringP2P",
228
+ "task_type": "Clustering",
229
+ "status": "completed",
230
+ "wall_seconds": 4.555887212045491,
231
+ "error": null
232
+ },
233
+ {
234
+ "task_index": 10,
235
+ "task_name": "RuSciBenchGRNTIClusteringP2P",
236
+ "task_type": "Clustering",
237
+ "status": "completed",
238
+ "wall_seconds": 3.001988342963159,
239
+ "error": null
240
+ },
241
+ {
242
+ "task_index": 11,
243
+ "task_name": "RuSciBenchOECDClusteringP2P",
244
+ "task_type": "Clustering",
245
+ "status": "completed",
246
+ "wall_seconds": 3.172198075102642,
247
+ "error": null
248
+ },
249
+ {
250
+ "task_index": 12,
251
+ "task_name": "CEDRClassification",
252
+ "task_type": "MultilabelClassification",
253
+ "status": "completed",
254
+ "wall_seconds": 9.170258222846314,
255
+ "error": null
256
+ },
257
+ {
258
+ "task_index": 13,
259
+ "task_name": "SensitiveTopicsClassification",
260
+ "task_type": "MultilabelClassification",
261
+ "status": "completed",
262
+ "wall_seconds": 23.70083056902513,
263
+ "error": null
264
+ },
265
+ {
266
+ "task_index": 14,
267
+ "task_name": "TERRa",
268
+ "task_type": "PairClassification",
269
+ "status": "completed",
270
+ "wall_seconds": 4.077696679858491,
271
+ "error": null
272
+ },
273
+ {
274
+ "task_index": 15,
275
+ "task_name": "MIRACLReranking",
276
+ "task_type": "Reranking",
277
+ "status": "completed",
278
+ "wall_seconds": 37.32648556190543,
279
+ "error": null
280
+ },
281
+ {
282
+ "task_index": 16,
283
+ "task_name": "RuBQReranking",
284
+ "task_type": "Reranking",
285
+ "status": "completed",
286
+ "wall_seconds": 15.40629133186303,
287
+ "error": null
288
+ },
289
+ {
290
+ "task_index": 17,
291
+ "task_name": "MIRACLRetrievalHardNegatives.v2",
292
+ "task_type": "Retrieval",
293
+ "status": "completed",
294
+ "wall_seconds": 58.23417071998119,
295
+ "error": null
296
+ },
297
+ {
298
+ "task_index": 18,
299
+ "task_name": "RiaNewsRetrievalHardNegatives.v2",
300
+ "task_type": "Retrieval",
301
+ "status": "completed",
302
+ "wall_seconds": 98.79431018303148,
303
+ "error": null
304
+ },
305
+ {
306
+ "task_index": 19,
307
+ "task_name": "RuBQRetrieval",
308
+ "task_type": "Retrieval",
309
+ "status": "completed",
310
+ "wall_seconds": 19.59109605406411,
311
+ "error": null
312
+ },
313
+ {
314
+ "task_index": 20,
315
+ "task_name": "RUParaPhraserSTS",
316
+ "task_type": "STS",
317
+ "status": "completed",
318
+ "wall_seconds": 3.623143167933449,
319
+ "error": null
320
+ },
321
+ {
322
+ "task_index": 21,
323
+ "task_name": "STS22",
324
+ "task_type": "STS",
325
+ "status": "completed",
326
+ "wall_seconds": 14.920513873919845,
327
+ "error": null
328
+ },
329
+ {
330
+ "task_index": 22,
331
+ "task_name": "RuSTSBenchmarkSTS",
332
+ "task_type": "STS",
333
+ "status": "completed",
334
+ "wall_seconds": 3.4421901470050216,
335
+ "error": null
336
+ }
337
+ ],
338
+ "model_name": "BorisTM/starse-512",
339
+ "run_name": "starse512_binary",
340
+ "total_wall_seconds": 610.5851712659933
341
+ }
results/mteb_rus_v1_1/raw_task_results/000_GeoreviewClassification.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "GeoreviewClassification": {
3
+ "dataset_revision": "fb29d6137c54550329411023aee45a8ff29f0c62",
4
+ "task_name": "GeoreviewClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.3798828125,
12
+ "f1": 0.3630554993282436,
13
+ "f1_weighted": 0.3631145095589351,
14
+ "precision": 0.3629432804686329,
15
+ "precision_weighted": 0.3629628318997826,
16
+ "recall": 0.37979485956228753,
17
+ "recall_weighted": 0.3798828125,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.38671875,
23
+ "f1": 0.37940968616391907,
24
+ "f1_weighted": 0.37934476944076007,
25
+ "precision": 0.37715774835535587,
26
+ "precision_weighted": 0.3771246579048203,
27
+ "recall": 0.3868173415230485,
28
+ "recall_weighted": 0.38671875,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.37841796875,
34
+ "f1": 0.3623365978871814,
35
+ "f1_weighted": 0.36230182483054285,
36
+ "precision": 0.3546509671102195,
37
+ "precision_weighted": 0.35462244683629807,
38
+ "recall": 0.3784519053014491,
39
+ "recall_weighted": 0.37841796875,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.3828125,
45
+ "f1": 0.37362660488190136,
46
+ "f1_weighted": 0.37361397190677326,
47
+ "precision": 0.3710129446484388,
48
+ "precision_weighted": 0.37099550180751234,
49
+ "recall": 0.3828254517263999,
50
+ "recall_weighted": 0.3828125,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.38037109375,
56
+ "f1": 0.3660399289281012,
57
+ "f1_weighted": 0.36603021337090796,
58
+ "precision": 0.38126555895601877,
59
+ "precision_weighted": 0.3812808096606408,
60
+ "recall": 0.38038284930526567,
61
+ "recall_weighted": 0.38037109375,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.34716796875,
67
+ "f1": 0.33672633938025165,
68
+ "f1_weighted": 0.3366453064920357,
69
+ "precision": 0.3370915728204227,
70
+ "precision_weighted": 0.33703378378252347,
71
+ "recall": 0.3472681734152305,
72
+ "recall_weighted": 0.34716796875,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.3466796875,
78
+ "f1": 0.3348639542162711,
79
+ "f1_weighted": 0.33480748202277133,
80
+ "precision": 0.3467430522198766,
81
+ "precision_weighted": 0.3466996893976442,
82
+ "recall": 0.3467720197984376,
83
+ "recall_weighted": 0.3466796875,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.35791015625,
89
+ "f1": 0.3548082152793765,
90
+ "f1_weighted": 0.35475885799074275,
91
+ "precision": 0.37537885222057354,
92
+ "precision_weighted": 0.375379166129309,
93
+ "recall": 0.35800107340926707,
94
+ "recall_weighted": 0.35791015625,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.34228515625,
100
+ "f1": 0.33495010934431346,
101
+ "f1_weighted": 0.3349097638048432,
102
+ "precision": 0.33375468738892444,
103
+ "precision_weighted": 0.3337362862322346,
104
+ "recall": 0.3423483809410221,
105
+ "recall_weighted": 0.34228515625,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.3984375,
111
+ "f1": 0.38512707597586365,
112
+ "f1_weighted": 0.38508569436339957,
113
+ "precision": 0.38097237674817375,
114
+ "precision_weighted": 0.3809604772212143,
115
+ "recall": 0.39850796111873094,
116
+ "recall_weighted": 0.3984375,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.370068359375,
122
+ "f1": 0.3590944011385423,
123
+ "f1_weighted": 0.35906123937817114,
124
+ "precision": 0.3620971040936637,
125
+ "precision_weighted": 0.36207956508719796,
126
+ "recall": 0.3701170016101139,
127
+ "recall_weighted": 0.370068359375,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.370068359375,
131
+ "hf_subset": "default",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ]
137
+ },
138
+ "evaluation_time": 15.229206085205078,
139
+ "kg_co2_emissions": null,
140
+ "date": null
141
+ }
142
+ }
results/mteb_rus_v1_1/raw_task_results/001_HeadlineClassification.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "HeadlineClassification": {
3
+ "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb",
4
+ "task_name": "HeadlineClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.71826171875,
12
+ "f1": 0.7195508399041896,
13
+ "f1_weighted": 0.7195608263103151,
14
+ "precision": 0.7274010583130398,
15
+ "precision_weighted": 0.7274045042710261,
16
+ "recall": 0.7182507016972212,
17
+ "recall_weighted": 0.71826171875,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.73046875,
23
+ "f1": 0.7316832842105533,
24
+ "f1_weighted": 0.7316868873504383,
25
+ "precision": 0.7378970085893796,
26
+ "precision_weighted": 0.7378545748012528,
27
+ "recall": 0.7304139299045921,
28
+ "recall_weighted": 0.73046875,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.72412109375,
34
+ "f1": 0.7235149855788694,
35
+ "f1_weighted": 0.7235251200082926,
36
+ "precision": 0.7266423500946732,
37
+ "precision_weighted": 0.7266331576403909,
38
+ "recall": 0.7240929384392881,
39
+ "recall_weighted": 0.72412109375,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.751953125,
45
+ "f1": 0.7534388300709226,
46
+ "f1_weighted": 0.7534446990480196,
47
+ "precision": 0.7567360564041646,
48
+ "precision_weighted": 0.7567364039966475,
49
+ "recall": 0.7519421721459073,
50
+ "recall_weighted": 0.751953125,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.75537109375,
56
+ "f1": 0.7561689428173093,
57
+ "f1_weighted": 0.7561621069755303,
58
+ "precision": 0.762645001004692,
59
+ "precision_weighted": 0.7626302877727491,
60
+ "recall": 0.7553706276117144,
61
+ "recall_weighted": 0.75537109375,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.78173828125,
67
+ "f1": 0.7822709621688025,
68
+ "f1_weighted": 0.7822632045125373,
69
+ "precision": 0.7857380707039189,
70
+ "precision_weighted": 0.7857077293385211,
71
+ "recall": 0.7817221450498192,
72
+ "recall_weighted": 0.78173828125,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.7451171875,
78
+ "f1": 0.745263108364393,
79
+ "f1_weighted": 0.7452466312520176,
80
+ "precision": 0.7464828771142574,
81
+ "precision_weighted": 0.7464480410302737,
82
+ "recall": 0.7451152727044069,
83
+ "recall_weighted": 0.7451171875,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.736328125,
89
+ "f1": 0.7364142355221434,
90
+ "f1_weighted": 0.73642032675172,
91
+ "precision": 0.7389678584361873,
92
+ "precision_weighted": 0.7389735261540632,
93
+ "recall": 0.7363219061011931,
94
+ "recall_weighted": 0.736328125,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.7275390625,
100
+ "f1": 0.7251463301353779,
101
+ "f1_weighted": 0.7251529900317777,
102
+ "precision": 0.7314349490877571,
103
+ "precision_weighted": 0.7313848293065623,
104
+ "recall": 0.7274742329920598,
105
+ "recall_weighted": 0.7275390625,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.7578125,
111
+ "f1": 0.7579074155576033,
112
+ "f1_weighted": 0.7579089693317103,
113
+ "precision": 0.7616356054726828,
114
+ "precision_weighted": 0.7616351090479117,
115
+ "recall": 0.7578101330223571,
116
+ "recall_weighted": 0.7578125,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.74287109375,
122
+ "f1": 0.7431358934330164,
123
+ "f1_weighted": 0.7431371761572358,
124
+ "precision": 0.7475580835220752,
125
+ "precision_weighted": 0.7475408163359398,
126
+ "recall": 0.742851405966856,
127
+ "recall_weighted": 0.74287109375,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.74287109375,
131
+ "hf_subset": "default",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ]
137
+ },
138
+ "evaluation_time": 20.96179699897766,
139
+ "kg_co2_emissions": null,
140
+ "date": null
141
+ }
142
+ }
results/mteb_rus_v1_1/raw_task_results/002_InappropriatenessClassification.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "InappropriatenessClassification": {
3
+ "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285",
4
+ "task_name": "InappropriatenessClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.6142578125,
12
+ "f1": 0.6136683171342991,
13
+ "f1_weighted": 0.6136683171342991,
14
+ "precision": 0.6149594694178334,
15
+ "precision_weighted": 0.6149594694178334,
16
+ "recall": 0.6142578125,
17
+ "recall_weighted": 0.6142578125,
18
+ "ap": 0.5712900969941737,
19
+ "ap_weighted": 0.5712900969941737
20
+ },
21
+ {
22
+ "accuracy": 0.61962890625,
23
+ "f1": 0.6191450175758986,
24
+ "f1_weighted": 0.6191450175758986,
25
+ "precision": 0.6202399815192375,
26
+ "precision_weighted": 0.6202399815192375,
27
+ "recall": 0.61962890625,
28
+ "recall_weighted": 0.61962890625,
29
+ "ap": 0.5731731960745214,
30
+ "ap_weighted": 0.5731731960745214
31
+ },
32
+ {
33
+ "accuracy": 0.5693359375,
34
+ "f1": 0.5630312923642915,
35
+ "f1_weighted": 0.5630312923642915,
36
+ "precision": 0.5735825759569257,
37
+ "precision_weighted": 0.5735825759569257,
38
+ "recall": 0.5693359375,
39
+ "recall_weighted": 0.5693359375,
40
+ "ap": 0.5409955414524421,
41
+ "ap_weighted": 0.5409955414524421
42
+ },
43
+ {
44
+ "accuracy": 0.55615234375,
45
+ "f1": 0.5550702265216703,
46
+ "f1_weighted": 0.5550702265216703,
47
+ "precision": 0.5567039845913085,
48
+ "precision_weighted": 0.5567039845913085,
49
+ "recall": 0.55615234375,
50
+ "recall_weighted": 0.55615234375,
51
+ "ap": 0.5315742864639762,
52
+ "ap_weighted": 0.5315742864639762
53
+ },
54
+ {
55
+ "accuracy": 0.57080078125,
56
+ "f1": 0.5585625681856135,
57
+ "f1_weighted": 0.5585625681856135,
58
+ "precision": 0.5796314471277868,
59
+ "precision_weighted": 0.5796314471277868,
60
+ "recall": 0.57080078125,
61
+ "recall_weighted": 0.57080078125,
62
+ "ap": 0.5429158469070278,
63
+ "ap_weighted": 0.5429158469070278
64
+ },
65
+ {
66
+ "accuracy": 0.5751953125,
67
+ "f1": 0.5750165770630692,
68
+ "f1_weighted": 0.5750165770630692,
69
+ "precision": 0.5753220253493464,
70
+ "precision_weighted": 0.5753220253493464,
71
+ "recall": 0.5751953125,
72
+ "recall_weighted": 0.5751953125,
73
+ "ap": 0.5434938263747454,
74
+ "ap_weighted": 0.5434938263747454
75
+ },
76
+ {
77
+ "accuracy": 0.62158203125,
78
+ "f1": 0.6209594702220821,
79
+ "f1_weighted": 0.6209594702220821,
80
+ "precision": 0.6223860910235033,
81
+ "precision_weighted": 0.6223860910235033,
82
+ "recall": 0.62158203125,
83
+ "recall_weighted": 0.62158203125,
84
+ "ap": 0.5744648755081301,
85
+ "ap_weighted": 0.5744648755081301
86
+ },
87
+ {
88
+ "accuracy": 0.595703125,
89
+ "f1": 0.5896864111498258,
90
+ "f1_weighted": 0.5896864111498258,
91
+ "precision": 0.6016663424755235,
92
+ "precision_weighted": 0.6016663424755235,
93
+ "recall": 0.595703125,
94
+ "recall_weighted": 0.595703125,
95
+ "ap": 0.5552249164701258,
96
+ "ap_weighted": 0.5552249164701258
97
+ },
98
+ {
99
+ "accuracy": 0.5908203125,
100
+ "f1": 0.589842420627182,
101
+ "f1_weighted": 0.589842420627182,
102
+ "precision": 0.5916947820862412,
103
+ "precision_weighted": 0.5916947820862412,
104
+ "recall": 0.5908203125,
105
+ "recall_weighted": 0.5908203125,
106
+ "ap": 0.5545511617288961,
107
+ "ap_weighted": 0.5545511617288961
108
+ },
109
+ {
110
+ "accuracy": 0.60107421875,
111
+ "f1": 0.6009706156254807,
112
+ "f1_weighted": 0.6009706156254807,
113
+ "precision": 0.6011792986452338,
114
+ "precision_weighted": 0.6011792986452338,
115
+ "recall": 0.60107421875,
116
+ "recall_weighted": 0.60107421875,
117
+ "ap": 0.561093296701564,
118
+ "ap_weighted": 0.561093296701564
119
+ }
120
+ ],
121
+ "accuracy": 0.591455078125,
122
+ "f1": 0.5885952916469412,
123
+ "f1_weighted": 0.5885952916469412,
124
+ "precision": 0.593736599819294,
125
+ "precision_weighted": 0.593736599819294,
126
+ "recall": 0.591455078125,
127
+ "recall_weighted": 0.591455078125,
128
+ "ap": 0.5548777044675602,
129
+ "ap_weighted": 0.5548777044675602,
130
+ "main_score": 0.591455078125,
131
+ "hf_subset": "default",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ]
137
+ },
138
+ "evaluation_time": 26.993356466293335,
139
+ "kg_co2_emissions": null,
140
+ "date": null
141
+ }
142
+ }
results/mteb_rus_v1_1/raw_task_results/003_KinopoiskClassification.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "KinopoiskClassification": {
3
+ "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24",
4
+ "task_name": "KinopoiskClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.45866666666666667,
12
+ "f1": 0.4563191259588004,
13
+ "f1_weighted": 0.45631912595880036,
14
+ "precision": 0.45697670332820534,
15
+ "precision_weighted": 0.4569767033282053,
16
+ "recall": 0.4586666666666666,
17
+ "recall_weighted": 0.45866666666666667,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.5086666666666667,
23
+ "f1": 0.4908922698972313,
24
+ "f1_weighted": 0.4908922698972312,
25
+ "precision": 0.4900512072852507,
26
+ "precision_weighted": 0.4900512072852507,
27
+ "recall": 0.5086666666666667,
28
+ "recall_weighted": 0.5086666666666667,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.5166666666666667,
34
+ "f1": 0.49862326800175544,
35
+ "f1_weighted": 0.49862326800175544,
36
+ "precision": 0.4971725439703894,
37
+ "precision_weighted": 0.49717254397038946,
38
+ "recall": 0.5166666666666667,
39
+ "recall_weighted": 0.5166666666666667,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.5106666666666667,
45
+ "f1": 0.5019044128790026,
46
+ "f1_weighted": 0.5019044128790026,
47
+ "precision": 0.4974491785018101,
48
+ "precision_weighted": 0.4974491785018101,
49
+ "recall": 0.5106666666666667,
50
+ "recall_weighted": 0.5106666666666667,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.46266666666666667,
56
+ "f1": 0.46396298829609367,
57
+ "f1_weighted": 0.46396298829609367,
58
+ "precision": 0.4728845924334646,
59
+ "precision_weighted": 0.4728845924334646,
60
+ "recall": 0.4626666666666666,
61
+ "recall_weighted": 0.46266666666666667,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.49533333333333335,
67
+ "f1": 0.4939937202536549,
68
+ "f1_weighted": 0.49399372025365496,
69
+ "precision": 0.49658477178862653,
70
+ "precision_weighted": 0.4965847717886265,
71
+ "recall": 0.49533333333333335,
72
+ "recall_weighted": 0.49533333333333335,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.5073333333333333,
78
+ "f1": 0.48720646585578525,
79
+ "f1_weighted": 0.4872064658557853,
80
+ "precision": 0.4881326462271245,
81
+ "precision_weighted": 0.4881326462271245,
82
+ "recall": 0.5073333333333333,
83
+ "recall_weighted": 0.5073333333333333,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.5073333333333333,
89
+ "f1": 0.48199402222220766,
90
+ "f1_weighted": 0.48199402222220766,
91
+ "precision": 0.4890258088035164,
92
+ "precision_weighted": 0.4890258088035165,
93
+ "recall": 0.5073333333333334,
94
+ "recall_weighted": 0.5073333333333333,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.538,
100
+ "f1": 0.5197333099650787,
101
+ "f1_weighted": 0.5197333099650787,
102
+ "precision": 0.5172297800770688,
103
+ "precision_weighted": 0.5172297800770689,
104
+ "recall": 0.5379999999999999,
105
+ "recall_weighted": 0.538,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.5186666666666667,
111
+ "f1": 0.49325658633184927,
112
+ "f1_weighted": 0.49325658633184927,
113
+ "precision": 0.49992321176999455,
114
+ "precision_weighted": 0.49992321176999455,
115
+ "recall": 0.5186666666666667,
116
+ "recall_weighted": 0.5186666666666667,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.5024,
122
+ "f1": 0.4887886169661459,
123
+ "f1_weighted": 0.4887886169661459,
124
+ "precision": 0.4905430444185451,
125
+ "precision_weighted": 0.4905430444185451,
126
+ "recall": 0.5024,
127
+ "recall_weighted": 0.5024,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.5024,
131
+ "hf_subset": "default",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ]
137
+ },
138
+ "evaluation_time": 8.922414064407349,
139
+ "kg_co2_emissions": null,
140
+ "date": null
141
+ }
142
+ }
results/mteb_rus_v1_1/raw_task_results/004_MassiveIntentClassification.json ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "MassiveIntentClassification": {
3
+ "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6",
4
+ "task_name": "MassiveIntentClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "validation": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.5892769306443679,
12
+ "f1": 0.5616443859463832,
13
+ "f1_weighted": 0.5850211668025836,
14
+ "precision": 0.5572762672108562,
15
+ "precision_weighted": 0.6424894265463587,
16
+ "recall": 0.6470691396639573,
17
+ "recall_weighted": 0.5892769306443679,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.6212493851451057,
23
+ "f1": 0.5635058949858203,
24
+ "f1_weighted": 0.6179352777843398,
25
+ "precision": 0.5473772201728336,
26
+ "precision_weighted": 0.6592379648856228,
27
+ "recall": 0.6421070988408062,
28
+ "recall_weighted": 0.6212493851451057,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.58780127889818,
34
+ "f1": 0.5444855012781142,
35
+ "f1_weighted": 0.5760624830129081,
36
+ "precision": 0.5309296466608094,
37
+ "precision_weighted": 0.6262338967106744,
38
+ "recall": 0.643330028118707,
39
+ "recall_weighted": 0.58780127889818,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.6010821446138711,
45
+ "f1": 0.5503595792495624,
46
+ "f1_weighted": 0.5918423785364821,
47
+ "precision": 0.5438455418278012,
48
+ "precision_weighted": 0.6496497907182278,
49
+ "recall": 0.6380820248383081,
50
+ "recall_weighted": 0.6010821446138711,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.6123954746679784,
56
+ "f1": 0.5531112634218686,
57
+ "f1_weighted": 0.5993287195242528,
58
+ "precision": 0.5580837213236738,
59
+ "precision_weighted": 0.6445664448009749,
60
+ "recall": 0.6373331596460108,
61
+ "recall_weighted": 0.6123954746679784,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.5892769306443679,
67
+ "f1": 0.5509655658909292,
68
+ "f1_weighted": 0.5832270714837751,
69
+ "precision": 0.5514633307149618,
70
+ "precision_weighted": 0.6693503881657102,
71
+ "recall": 0.6480784783846363,
72
+ "recall_weighted": 0.5892769306443679,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.602065912444663,
78
+ "f1": 0.5760345737836746,
79
+ "f1_weighted": 0.5926078146088083,
80
+ "precision": 0.5759154371358756,
81
+ "precision_weighted": 0.6471751713373017,
82
+ "recall": 0.6489378200887612,
83
+ "recall_weighted": 0.602065912444663,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.5833743236596163,
89
+ "f1": 0.5339404202463032,
90
+ "f1_weighted": 0.5746337696180803,
91
+ "precision": 0.5433771815821525,
92
+ "precision_weighted": 0.6443500945120295,
93
+ "recall": 0.6160099313000706,
94
+ "recall_weighted": 0.5833743236596163,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.5725528775209051,
100
+ "f1": 0.5356405760184749,
101
+ "f1_weighted": 0.5544935682395975,
102
+ "precision": 0.5412724274326296,
103
+ "precision_weighted": 0.6455358717980654,
104
+ "recall": 0.6373171478581509,
105
+ "recall_weighted": 0.5725528775209051,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.6310870634530251,
111
+ "f1": 0.5768855541663143,
112
+ "f1_weighted": 0.6238808636518269,
113
+ "precision": 0.5739757652872747,
114
+ "precision_weighted": 0.6856003582027603,
115
+ "recall": 0.6609986954875315,
116
+ "recall_weighted": 0.6310870634530251,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.599016232169208,
122
+ "f1": 0.5546573314987444,
123
+ "f1_weighted": 0.5899033113262655,
124
+ "precision": 0.5523516539348868,
125
+ "precision_weighted": 0.6514189407677726,
126
+ "recall": 0.6419263524226941,
127
+ "recall_weighted": 0.599016232169208,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.599016232169208,
131
+ "hf_subset": "ru",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ],
137
+ "test": [
138
+ {
139
+ "scores_per_experiment": [
140
+ {
141
+ "accuracy": 0.5995292535305985,
142
+ "f1": 0.5804569124457889,
143
+ "f1_weighted": 0.5934999369723852,
144
+ "precision": 0.5754405144758988,
145
+ "precision_weighted": 0.6513882786934158,
146
+ "recall": 0.6793840627132954,
147
+ "recall_weighted": 0.5995292535305985,
148
+ "ap": null,
149
+ "ap_weighted": null
150
+ },
151
+ {
152
+ "accuracy": 0.6240753194351042,
153
+ "f1": 0.5897907484311097,
154
+ "f1_weighted": 0.6210410408125037,
155
+ "precision": 0.5807395217719976,
156
+ "precision_weighted": 0.6720104602821323,
157
+ "recall": 0.6641018145075254,
158
+ "recall_weighted": 0.6240753194351042,
159
+ "ap": null,
160
+ "ap_weighted": null
161
+ },
162
+ {
163
+ "accuracy": 0.5756556825823806,
164
+ "f1": 0.5465761353998634,
165
+ "f1_weighted": 0.5622415316747825,
166
+ "precision": 0.540371284257422,
167
+ "precision_weighted": 0.6248137131873444,
168
+ "recall": 0.6385638618918804,
169
+ "recall_weighted": 0.5756556825823806,
170
+ "ap": null,
171
+ "ap_weighted": null
172
+ },
173
+ {
174
+ "accuracy": 0.6069266980497646,
175
+ "f1": 0.5678043760733001,
176
+ "f1_weighted": 0.5996736009748628,
177
+ "precision": 0.5602740405961768,
178
+ "precision_weighted": 0.6576645173079971,
179
+ "recall": 0.6720131876933249,
180
+ "recall_weighted": 0.6069266980497646,
181
+ "ap": null,
182
+ "ap_weighted": null
183
+ },
184
+ {
185
+ "accuracy": 0.6059179556153329,
186
+ "f1": 0.5650234118675739,
187
+ "f1_weighted": 0.5934996442912811,
188
+ "precision": 0.559439572398503,
189
+ "precision_weighted": 0.6476915273550977,
190
+ "recall": 0.6761449523037184,
191
+ "recall_weighted": 0.6059179556153329,
192
+ "ap": null,
193
+ "ap_weighted": null
194
+ },
195
+ {
196
+ "accuracy": 0.5786819098856758,
197
+ "f1": 0.5532436420993944,
198
+ "f1_weighted": 0.5730914420306,
199
+ "precision": 0.5568113710338242,
200
+ "precision_weighted": 0.6585696319699343,
201
+ "recall": 0.6546779922281826,
202
+ "recall_weighted": 0.5786819098856758,
203
+ "ap": null,
204
+ "ap_weighted": null
205
+ },
206
+ {
207
+ "accuracy": 0.5938130464021519,
208
+ "f1": 0.5787712005343617,
209
+ "f1_weighted": 0.5862038602074827,
210
+ "precision": 0.5790487245657928,
211
+ "precision_weighted": 0.6430091450023193,
212
+ "recall": 0.6733193599944435,
213
+ "recall_weighted": 0.5938130464021519,
214
+ "ap": null,
215
+ "ap_weighted": null
216
+ },
217
+ {
218
+ "accuracy": 0.5857431069266981,
219
+ "f1": 0.555874784360556,
220
+ "f1_weighted": 0.5754637971677626,
221
+ "precision": 0.5649896553743251,
222
+ "precision_weighted": 0.6660283400923609,
223
+ "recall": 0.6445964474229168,
224
+ "recall_weighted": 0.5857431069266981,
225
+ "ap": null,
226
+ "ap_weighted": null
227
+ },
228
+ {
229
+ "accuracy": 0.5679219905850706,
230
+ "f1": 0.5398939696415314,
231
+ "f1_weighted": 0.5467697770626171,
232
+ "precision": 0.5526066622638467,
233
+ "precision_weighted": 0.6467331589829381,
234
+ "recall": 0.648629127445207,
235
+ "recall_weighted": 0.5679219905850706,
236
+ "ap": null,
237
+ "ap_weighted": null
238
+ },
239
+ {
240
+ "accuracy": 0.6240753194351042,
241
+ "f1": 0.5820685927861012,
242
+ "f1_weighted": 0.6171232888676796,
243
+ "precision": 0.5802587798790676,
244
+ "precision_weighted": 0.6701221400147399,
245
+ "recall": 0.6767301757356884,
246
+ "recall_weighted": 0.6240753194351042,
247
+ "ap": null,
248
+ "ap_weighted": null
249
+ }
250
+ ],
251
+ "accuracy": 0.5962340282447882,
252
+ "f1": 0.565950377363958,
253
+ "f1_weighted": 0.5868607920061957,
254
+ "precision": 0.5649980126616854,
255
+ "precision_weighted": 0.653803091288828,
256
+ "recall": 0.6628160981936182,
257
+ "recall_weighted": 0.5962340282447882,
258
+ "ap": NaN,
259
+ "ap_weighted": NaN,
260
+ "main_score": 0.5962340282447882,
261
+ "hf_subset": "ru",
262
+ "languages": [
263
+ "rus-Cyrl"
264
+ ]
265
+ }
266
+ ]
267
+ },
268
+ "evaluation_time": 49.436988830566406,
269
+ "kg_co2_emissions": null,
270
+ "date": null
271
+ }
272
+ }
results/mteb_rus_v1_1/raw_task_results/005_MassiveScenarioClassification.json ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "MassiveScenarioClassification": {
3
+ "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8",
4
+ "task_name": "MassiveScenarioClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "validation": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.6999508116084604,
12
+ "f1": 0.6868595955620889,
13
+ "f1_weighted": 0.6972717339122584,
14
+ "precision": 0.6712164642720254,
15
+ "precision_weighted": 0.7275693869757998,
16
+ "recall": 0.7377451480133354,
17
+ "recall_weighted": 0.6999508116084604,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.6719134284308903,
23
+ "f1": 0.6664857398878772,
24
+ "f1_weighted": 0.6662152500475047,
25
+ "precision": 0.6459167116672926,
26
+ "precision_weighted": 0.7022067886660834,
27
+ "recall": 0.7287240197918224,
28
+ "recall_weighted": 0.6719134284308903,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.7063453025086079,
34
+ "f1": 0.6940628094920922,
35
+ "f1_weighted": 0.7035265108089608,
36
+ "precision": 0.6723547104016184,
37
+ "precision_weighted": 0.7326432105548938,
38
+ "recall": 0.7532134456819419,
39
+ "recall_weighted": 0.7063453025086079,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.6512543039842598,
45
+ "f1": 0.6455150999738029,
46
+ "f1_weighted": 0.641385405855911,
47
+ "precision": 0.6330718277874434,
48
+ "precision_weighted": 0.6848237205830243,
49
+ "recall": 0.7131776528019054,
50
+ "recall_weighted": 0.6512543039842598,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.7191342843089031,
56
+ "f1": 0.7093196748628738,
57
+ "f1_weighted": 0.7137578936624126,
58
+ "precision": 0.6910677468376876,
59
+ "precision_weighted": 0.7441918762149746,
60
+ "recall": 0.7623368923980859,
61
+ "recall_weighted": 0.7191342843089031,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.6665027053615347,
67
+ "f1": 0.6646439912225772,
68
+ "f1_weighted": 0.6622575406515033,
69
+ "precision": 0.6520727369162891,
70
+ "precision_weighted": 0.7085269323344242,
71
+ "recall": 0.7211635835798116,
72
+ "recall_weighted": 0.6665027053615347,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.6694540088539105,
78
+ "f1": 0.6662053528595121,
79
+ "f1_weighted": 0.6695549227576636,
80
+ "precision": 0.6536765268210527,
81
+ "precision_weighted": 0.6964304378836997,
82
+ "recall": 0.708539596468939,
83
+ "recall_weighted": 0.6694540088539105,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.6847024102311854,
89
+ "f1": 0.6751123641997661,
90
+ "f1_weighted": 0.6811190811821258,
91
+ "precision": 0.66355770836701,
92
+ "precision_weighted": 0.7070585043607563,
93
+ "recall": 0.7166044282537902,
94
+ "recall_weighted": 0.6847024102311854,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.6669945892769307,
100
+ "f1": 0.6671045936716526,
101
+ "f1_weighted": 0.6645778826327913,
102
+ "precision": 0.6512325685096961,
103
+ "precision_weighted": 0.6998701147255196,
104
+ "recall": 0.7206155136344641,
105
+ "recall_weighted": 0.6669945892769307,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.675848499754058,
111
+ "f1": 0.6742155292978049,
112
+ "f1_weighted": 0.6701022576466402,
113
+ "precision": 0.651686360656879,
114
+ "precision_weighted": 0.7083495237834815,
115
+ "recall": 0.7399625641708543,
116
+ "recall_weighted": 0.675848499754058,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.6812100344318741,
122
+ "f1": 0.6749524751030048,
123
+ "f1_weighted": 0.6769768479157772,
124
+ "precision": 0.6585853362236995,
125
+ "precision_weighted": 0.7111670496082658,
126
+ "recall": 0.730208284479495,
127
+ "recall_weighted": 0.6812100344318741,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.6812100344318741,
131
+ "hf_subset": "ru",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ],
137
+ "test": [
138
+ {
139
+ "scores_per_experiment": [
140
+ {
141
+ "accuracy": 0.6809011432414257,
142
+ "f1": 0.6716304153166262,
143
+ "f1_weighted": 0.6746696175026696,
144
+ "precision": 0.6549850000207862,
145
+ "precision_weighted": 0.7014740265997598,
146
+ "recall": 0.7223461298859947,
147
+ "recall_weighted": 0.6809011432414257,
148
+ "ap": null,
149
+ "ap_weighted": null
150
+ },
151
+ {
152
+ "accuracy": 0.6714862138533961,
153
+ "f1": 0.6675690417195249,
154
+ "f1_weighted": 0.6659915880401718,
155
+ "precision": 0.650529477130529,
156
+ "precision_weighted": 0.7018606940199623,
157
+ "recall": 0.7256926283122723,
158
+ "recall_weighted": 0.6714862138533961,
159
+ "ap": null,
160
+ "ap_weighted": null
161
+ },
162
+ {
163
+ "accuracy": 0.6896435776731674,
164
+ "f1": 0.6780035611791334,
165
+ "f1_weighted": 0.6842426698435998,
166
+ "precision": 0.6574986032700895,
167
+ "precision_weighted": 0.7112641575287346,
168
+ "recall": 0.7355257534973949,
169
+ "recall_weighted": 0.6896435776731674,
170
+ "ap": null,
171
+ "ap_weighted": null
172
+ },
173
+ {
174
+ "accuracy": 0.6661062542030934,
175
+ "f1": 0.6626916287533263,
176
+ "f1_weighted": 0.6568922171672475,
177
+ "precision": 0.6518174721313569,
178
+ "precision_weighted": 0.7001207511387411,
179
+ "recall": 0.725379623985565,
180
+ "recall_weighted": 0.6661062542030934,
181
+ "ap": null,
182
+ "ap_weighted": null
183
+ },
184
+ {
185
+ "accuracy": 0.7057834566240753,
186
+ "f1": 0.6908466352728673,
187
+ "f1_weighted": 0.7004769541543812,
188
+ "precision": 0.6786672658910482,
189
+ "precision_weighted": 0.732015765266294,
190
+ "recall": 0.7421571947594202,
191
+ "recall_weighted": 0.7057834566240753,
192
+ "ap": null,
193
+ "ap_weighted": null
194
+ },
195
+ {
196
+ "accuracy": 0.6647612642905179,
197
+ "f1": 0.6561051672182338,
198
+ "f1_weighted": 0.6620616518040601,
199
+ "precision": 0.6497867866307178,
200
+ "precision_weighted": 0.7086430610741492,
201
+ "recall": 0.7074533201725927,
202
+ "recall_weighted": 0.6647612642905179,
203
+ "ap": null,
204
+ "ap_weighted": null
205
+ },
206
+ {
207
+ "accuracy": 0.6698049764626766,
208
+ "f1": 0.6698442181374906,
209
+ "f1_weighted": 0.6687881512911439,
210
+ "precision": 0.6585233613079851,
211
+ "precision_weighted": 0.6949885354876035,
212
+ "recall": 0.7132195967104223,
213
+ "recall_weighted": 0.6698049764626766,
214
+ "ap": null,
215
+ "ap_weighted": null
216
+ },
217
+ {
218
+ "accuracy": 0.6815736381977135,
219
+ "f1": 0.6760992045356027,
220
+ "f1_weighted": 0.6785330181754762,
221
+ "precision": 0.6680219783594147,
222
+ "precision_weighted": 0.709040191275429,
223
+ "recall": 0.721104147698944,
224
+ "recall_weighted": 0.6815736381977135,
225
+ "ap": null,
226
+ "ap_weighted": null
227
+ },
228
+ {
229
+ "accuracy": 0.6761936785474109,
230
+ "f1": 0.6801200157124616,
231
+ "f1_weighted": 0.6742618678995702,
232
+ "precision": 0.6672117167475704,
233
+ "precision_weighted": 0.711716363672244,
234
+ "recall": 0.731802663012899,
235
+ "recall_weighted": 0.6761936785474109,
236
+ "ap": null,
237
+ "ap_weighted": null
238
+ },
239
+ {
240
+ "accuracy": 0.6691324815063887,
241
+ "f1": 0.6669263922435549,
242
+ "f1_weighted": 0.6640069045530093,
243
+ "precision": 0.649015456284421,
244
+ "precision_weighted": 0.7016719405564035,
245
+ "recall": 0.7272714451676459,
246
+ "recall_weighted": 0.6691324815063887,
247
+ "ap": null,
248
+ "ap_weighted": null
249
+ }
250
+ ],
251
+ "accuracy": 0.6775386684599866,
252
+ "f1": 0.671983628008882,
253
+ "f1_weighted": 0.672992464043133,
254
+ "precision": 0.6586057117773919,
255
+ "precision_weighted": 0.7072795486619321,
256
+ "recall": 0.7251952503203152,
257
+ "recall_weighted": 0.6775386684599866,
258
+ "ap": NaN,
259
+ "ap_weighted": NaN,
260
+ "main_score": 0.6775386684599866,
261
+ "hf_subset": "ru",
262
+ "languages": [
263
+ "rus-Cyrl"
264
+ ]
265
+ }
266
+ ]
267
+ },
268
+ "evaluation_time": 19.019729137420654,
269
+ "kg_co2_emissions": null,
270
+ "date": null
271
+ }
272
+ }
results/mteb_rus_v1_1/raw_task_results/006_RuReviewsClassification.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuReviewsClassification": {
3
+ "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a",
4
+ "task_name": "RuReviewsClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.57763671875,
12
+ "f1": 0.564782532625108,
13
+ "f1_weighted": 0.5647645763563781,
14
+ "precision": 0.5721395828561534,
15
+ "precision_weighted": 0.5721350634391704,
16
+ "recall": 0.5776589395585287,
17
+ "recall_weighted": 0.57763671875,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.53662109375,
23
+ "f1": 0.5328708993641266,
24
+ "f1_weighted": 0.5328598602622469,
25
+ "precision": 0.5324568914868205,
26
+ "precision_weighted": 0.5324609928902979,
27
+ "recall": 0.5366475886241626,
28
+ "recall_weighted": 0.53662109375,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.560546875,
34
+ "f1": 0.5616707111832123,
35
+ "f1_weighted": 0.5616625467103604,
36
+ "precision": 0.5689644835816293,
37
+ "precision_weighted": 0.5689256714493847,
38
+ "recall": 0.5605280596070754,
39
+ "recall_weighted": 0.560546875,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.603515625,
45
+ "f1": 0.6063791463972549,
46
+ "f1_weighted": 0.6063879181317524,
47
+ "precision": 0.613600190969047,
48
+ "precision_weighted": 0.6136006312904683,
49
+ "recall": 0.6034973071765212,
50
+ "recall_weighted": 0.603515625,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.56201171875,
56
+ "f1": 0.5646666544558758,
57
+ "f1_weighted": 0.564650827484433,
58
+ "precision": 0.5749848654613138,
59
+ "precision_weighted": 0.5749753479424053,
60
+ "recall": 0.5620301155416633,
61
+ "recall_weighted": 0.56201171875,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.513671875,
67
+ "f1": 0.5009253259678984,
68
+ "f1_weighted": 0.5009452407461348,
69
+ "precision": 0.4961173124935961,
70
+ "precision_weighted": 0.49614479185897753,
71
+ "recall": 0.51365661527188,
72
+ "recall_weighted": 0.513671875,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.52587890625,
78
+ "f1": 0.5197864719903427,
79
+ "f1_weighted": 0.5197946355583738,
80
+ "precision": 0.5216790333229695,
81
+ "precision_weighted": 0.5216762734809833,
82
+ "recall": 0.5258569733608698,
83
+ "recall_weighted": 0.52587890625,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.53466796875,
89
+ "f1": 0.5307460122968707,
90
+ "f1_weighted": 0.5307263619010079,
91
+ "precision": 0.532136566223869,
92
+ "precision_weighted": 0.5321039015509191,
93
+ "recall": 0.5346732330626913,
94
+ "recall_weighted": 0.53466796875,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.47998046875,
100
+ "f1": 0.47883162968809634,
101
+ "f1_weighted": 0.4788014983839691,
102
+ "precision": 0.48659255271064317,
103
+ "precision_weighted": 0.4865419251593408,
104
+ "recall": 0.47998952353554913,
105
+ "recall_weighted": 0.47998046875,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.58154296875,
111
+ "f1": 0.5782729100685041,
112
+ "f1_weighted": 0.5782590728320883,
113
+ "precision": 0.5884329619676204,
114
+ "precision_weighted": 0.5883808902900775,
115
+ "recall": 0.5815217780220377,
116
+ "recall_weighted": 0.58154296875,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.547607421875,
122
+ "f1": 0.543893229403729,
123
+ "f1_weighted": 0.5438852538366745,
124
+ "precision": 0.5487104441073662,
125
+ "precision_weighted": 0.5486945489352025,
126
+ "recall": 0.5476060133760978,
127
+ "recall_weighted": 0.547607421875,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.547607421875,
131
+ "hf_subset": "default",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ]
137
+ },
138
+ "evaluation_time": 22.69205951690674,
139
+ "kg_co2_emissions": null,
140
+ "date": null
141
+ }
142
+ }
results/mteb_rus_v1_1/raw_task_results/007_RuSciBenchGRNTIClassification.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuSciBenchGRNTIClassification": {
3
+ "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1",
4
+ "task_name": "RuSciBenchGRNTIClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.62158203125,
12
+ "f1": 0.6094752040184288,
13
+ "f1_weighted": 0.609592988840237,
14
+ "precision": 0.6411005562698776,
15
+ "precision_weighted": 0.6412242697100431,
16
+ "recall": 0.6215488972338289,
17
+ "recall_weighted": 0.62158203125,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.60888671875,
23
+ "f1": 0.5994955179581819,
24
+ "f1_weighted": 0.5995934541233474,
25
+ "precision": 0.6124201772141971,
26
+ "precision_weighted": 0.612488773769084,
27
+ "recall": 0.6088022954461311,
28
+ "recall_weighted": 0.60888671875,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.5849609375,
34
+ "f1": 0.5733875072198659,
35
+ "f1_weighted": 0.5735464380719696,
36
+ "precision": 0.5901158527044077,
37
+ "precision_weighted": 0.5902384373024531,
38
+ "recall": 0.5848032474744803,
39
+ "recall_weighted": 0.5849609375,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.62451171875,
45
+ "f1": 0.6174111119257878,
46
+ "f1_weighted": 0.6174660025984698,
47
+ "precision": 0.6302910641497895,
48
+ "precision_weighted": 0.6302427538239087,
49
+ "recall": 0.6243719257417888,
50
+ "recall_weighted": 0.62451171875,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.6025390625,
56
+ "f1": 0.601241154003654,
57
+ "f1_weighted": 0.6013252864633436,
58
+ "precision": 0.622191040730112,
59
+ "precision_weighted": 0.6222639125150169,
60
+ "recall": 0.6024818850161316,
61
+ "recall_weighted": 0.6025390625,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.5869140625,
67
+ "f1": 0.5735472967425049,
68
+ "f1_weighted": 0.5736298923402497,
69
+ "precision": 0.604188619471924,
70
+ "precision_weighted": 0.6042260709470446,
71
+ "recall": 0.5868527529486434,
72
+ "recall_weighted": 0.5869140625,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.61962890625,
78
+ "f1": 0.609456219408675,
79
+ "f1_weighted": 0.6096024681657325,
80
+ "precision": 0.6297775993646789,
81
+ "precision_weighted": 0.6298307348459458,
82
+ "recall": 0.6194266673718729,
83
+ "recall_weighted": 0.61962890625,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.591796875,
89
+ "f1": 0.5781453698332139,
90
+ "f1_weighted": 0.5782571576933815,
91
+ "precision": 0.6024461281170908,
92
+ "precision_weighted": 0.602431170059003,
93
+ "recall": 0.5915996720791241,
94
+ "recall_weighted": 0.591796875,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.5966796875,
100
+ "f1": 0.5875616151196533,
101
+ "f1_weighted": 0.5876679095166027,
102
+ "precision": 0.6129851256192232,
103
+ "precision_weighted": 0.6130770576007482,
104
+ "recall": 0.5965978209128894,
105
+ "recall_weighted": 0.5966796875,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.5908203125,
111
+ "f1": 0.5784667398476663,
112
+ "f1_weighted": 0.5785669329596651,
113
+ "precision": 0.5974208286756288,
114
+ "precision_weighted": 0.5974511281627516,
115
+ "recall": 0.5907005341936848,
116
+ "recall_weighted": 0.5908203125,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.60283203125,
122
+ "f1": 0.5928187736077632,
123
+ "f1_weighted": 0.5929248530772999,
124
+ "precision": 0.6142936992316931,
125
+ "precision_weighted": 0.6143474308735999,
126
+ "recall": 0.6027185698418575,
127
+ "recall_weighted": 0.60283203125,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.60283203125,
131
+ "hf_subset": "default",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ]
137
+ },
138
+ "evaluation_time": 32.56986856460571,
139
+ "kg_co2_emissions": null,
140
+ "date": null
141
+ }
142
+ }
results/mteb_rus_v1_1/raw_task_results/008_RuSciBenchOECDClassification.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuSciBenchOECDClassification": {
3
+ "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471",
4
+ "task_name": "RuSciBenchOECDClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.49951171875,
12
+ "f1": 0.4849894625059667,
13
+ "f1_weighted": 0.4850457581844529,
14
+ "precision": 0.5042713836558745,
15
+ "precision_weighted": 0.5044743392097785,
16
+ "recall": 0.4996114618746964,
17
+ "recall_weighted": 0.49951171875,
18
+ "ap": null,
19
+ "ap_weighted": null
20
+ },
21
+ {
22
+ "accuracy": 0.48388671875,
23
+ "f1": 0.4646437750503343,
24
+ "f1_weighted": 0.46460842235691835,
25
+ "precision": 0.4851745697019973,
26
+ "precision_weighted": 0.48524503507867794,
27
+ "recall": 0.4839450496079928,
28
+ "recall_weighted": 0.48388671875,
29
+ "ap": null,
30
+ "ap_weighted": null
31
+ },
32
+ {
33
+ "accuracy": 0.48193359375,
34
+ "f1": 0.46355882233437784,
35
+ "f1_weighted": 0.4635456796954545,
36
+ "precision": 0.4894114915929429,
37
+ "precision_weighted": 0.48947325229294303,
38
+ "recall": 0.482043988066329,
39
+ "recall_weighted": 0.48193359375,
40
+ "ap": null,
41
+ "ap_weighted": null
42
+ },
43
+ {
44
+ "accuracy": 0.4677734375,
45
+ "f1": 0.4430819818934167,
46
+ "f1_weighted": 0.4429682506781859,
47
+ "precision": 0.4533264813438359,
48
+ "precision_weighted": 0.4532636945866488,
49
+ "recall": 0.46798723374731155,
50
+ "recall_weighted": 0.4677734375,
51
+ "ap": null,
52
+ "ap_weighted": null
53
+ },
54
+ {
55
+ "accuracy": 0.48828125,
56
+ "f1": 0.4684474581236782,
57
+ "f1_weighted": 0.46841551960633726,
58
+ "precision": 0.4833818957914964,
59
+ "precision_weighted": 0.48351043652529285,
60
+ "recall": 0.4884062998681747,
61
+ "recall_weighted": 0.48828125,
62
+ "ap": null,
63
+ "ap_weighted": null
64
+ },
65
+ {
66
+ "accuracy": 0.49267578125,
67
+ "f1": 0.48015942363326597,
68
+ "f1_weighted": 0.4802016152070993,
69
+ "precision": 0.5038354866135104,
70
+ "precision_weighted": 0.503950823598361,
71
+ "recall": 0.4927426628737944,
72
+ "recall_weighted": 0.49267578125,
73
+ "ap": null,
74
+ "ap_weighted": null
75
+ },
76
+ {
77
+ "accuracy": 0.46728515625,
78
+ "f1": 0.4473925433287486,
79
+ "f1_weighted": 0.447473365154656,
80
+ "precision": 0.4730599929428712,
81
+ "precision_weighted": 0.47322208578050057,
82
+ "recall": 0.46732116838964827,
83
+ "recall_weighted": 0.46728515625,
84
+ "ap": null,
85
+ "ap_weighted": null
86
+ },
87
+ {
88
+ "accuracy": 0.49462890625,
89
+ "f1": 0.48234358111617626,
90
+ "f1_weighted": 0.4823560292113337,
91
+ "precision": 0.5005726173645294,
92
+ "precision_weighted": 0.5005571537894028,
93
+ "recall": 0.4945465898841323,
94
+ "recall_weighted": 0.49462890625,
95
+ "ap": null,
96
+ "ap_weighted": null
97
+ },
98
+ {
99
+ "accuracy": 0.46630859375,
100
+ "f1": 0.44575052479974886,
101
+ "f1_weighted": 0.4456807966404122,
102
+ "precision": 0.4831418583112417,
103
+ "precision_weighted": 0.48318782542025585,
104
+ "recall": 0.4663706376188164,
105
+ "recall_weighted": 0.46630859375,
106
+ "ap": null,
107
+ "ap_weighted": null
108
+ },
109
+ {
110
+ "accuracy": 0.4775390625,
111
+ "f1": 0.462662124461228,
112
+ "f1_weighted": 0.46256124946215127,
113
+ "precision": 0.4922022657975989,
114
+ "precision_weighted": 0.492379866476604,
115
+ "recall": 0.477804759592035,
116
+ "recall_weighted": 0.4775390625,
117
+ "ap": null,
118
+ "ap_weighted": null
119
+ }
120
+ ],
121
+ "accuracy": 0.481982421875,
122
+ "f1": 0.46430296972469415,
123
+ "f1_weighted": 0.46428566861970016,
124
+ "precision": 0.48683780431158985,
125
+ "precision_weighted": 0.48692645127584655,
126
+ "recall": 0.4820779851522931,
127
+ "recall_weighted": 0.481982421875,
128
+ "ap": NaN,
129
+ "ap_weighted": NaN,
130
+ "main_score": 0.481982421875,
131
+ "hf_subset": "default",
132
+ "languages": [
133
+ "rus-Cyrl"
134
+ ]
135
+ }
136
+ ]
137
+ },
138
+ "evaluation_time": 32.00241756439209,
139
+ "kg_co2_emissions": null,
140
+ "date": null
141
+ }
142
+ }
results/mteb_rus_v1_1/raw_task_results/009_GeoreviewClusteringP2P.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "GeoreviewClusteringP2P": {
3
+ "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec",
4
+ "task_name": "GeoreviewClusteringP2P",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "v_measures": {
10
+ "Level 0": [
11
+ 0.5386405684384751,
12
+ 0.5408487123068749,
13
+ 0.5440805367709383,
14
+ 0.5293764422813401,
15
+ 0.5406554540680786,
16
+ 0.513162638748291,
17
+ 0.5457372944948743,
18
+ 0.5490115866412916,
19
+ 0.5625817388460563,
20
+ 0.5354089633418472
21
+ ]
22
+ },
23
+ "v_measure": 0.5399503935938068,
24
+ "v_measure_std": 0.01223873312703622,
25
+ "main_score": 0.5399503935938068,
26
+ "hf_subset": "default",
27
+ "languages": [
28
+ "rus-Cyrl"
29
+ ]
30
+ }
31
+ ]
32
+ },
33
+ "evaluation_time": 2.040574789047241,
34
+ "kg_co2_emissions": null,
35
+ "date": null
36
+ }
37
+ }
results/mteb_rus_v1_1/raw_task_results/010_RuSciBenchGRNTIClusteringP2P.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuSciBenchGRNTIClusteringP2P": {
3
+ "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1",
4
+ "task_name": "RuSciBenchGRNTIClusteringP2P",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "v_measures": {
10
+ "Level 0": [
11
+ 0.5277523549010623,
12
+ 0.5367509277504748,
13
+ 0.5418916048338469,
14
+ 0.5479945155058874,
15
+ 0.5455098696152075,
16
+ 0.5317007842988902,
17
+ 0.5416995633440382,
18
+ 0.5564005914416973,
19
+ 0.5272012385117475,
20
+ 0.5378417444271099
21
+ ]
22
+ },
23
+ "v_measure": 0.5394743194629961,
24
+ "v_measure_std": 0.008745617474385223,
25
+ "main_score": 0.5394743194629961,
26
+ "hf_subset": "default",
27
+ "languages": [
28
+ "rus-Cyrl"
29
+ ]
30
+ }
31
+ ]
32
+ },
33
+ "evaluation_time": 1.9656360149383545,
34
+ "kg_co2_emissions": null,
35
+ "date": null
36
+ }
37
+ }
results/mteb_rus_v1_1/raw_task_results/011_RuSciBenchOECDClusteringP2P.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuSciBenchOECDClusteringP2P": {
3
+ "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471",
4
+ "task_name": "RuSciBenchOECDClusteringP2P",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "v_measures": {
10
+ "Level 0": [
11
+ 0.4726134739713615,
12
+ 0.46634365641563835,
13
+ 0.4812148514928899,
14
+ 0.4825253488474197,
15
+ 0.464761224393946,
16
+ 0.46076982950422535,
17
+ 0.4724791974536807,
18
+ 0.47945216061151785,
19
+ 0.48023169741184435,
20
+ 0.48493086380376693
21
+ ]
22
+ },
23
+ "v_measure": 0.47453223039062903,
24
+ "v_measure_std": 0.0079573496376099,
25
+ "main_score": 0.47453223039062903,
26
+ "hf_subset": "default",
27
+ "languages": [
28
+ "rus-Cyrl"
29
+ ]
30
+ }
31
+ ]
32
+ },
33
+ "evaluation_time": 2.1811301708221436,
34
+ "kg_co2_emissions": null,
35
+ "date": null
36
+ }
37
+ }
results/mteb_rus_v1_1/raw_task_results/012_CEDRClassification.json ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "CEDRClassification": {
3
+ "dataset_revision": "d872ceae33d049f0ef50e8abb9229b569ebfcf7d",
4
+ "task_name": "CEDRClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.4585547290116897,
12
+ "lrap": 0.7164718384697235,
13
+ "f1": 0.4112312384312468,
14
+ "hamming": 0.46493092454835283
15
+ },
16
+ {
17
+ "accuracy": 0.4479277364505845,
18
+ "lrap": 0.6573857598299792,
19
+ "f1": 0.31999326169913606,
20
+ "hamming": 0.45217853347502657
21
+ },
22
+ {
23
+ "accuracy": 0.3831030818278427,
24
+ "lrap": 0.6464930924548467,
25
+ "f1": 0.26361954702065804,
26
+ "hamming": 0.3854941551540914
27
+ },
28
+ {
29
+ "accuracy": 0.4351753453772582,
30
+ "lrap": 0.6727948990435818,
31
+ "f1": 0.3188686098479617,
32
+ "hamming": 0.448990435706695
33
+ },
34
+ {
35
+ "accuracy": 0.4038257173219979,
36
+ "lrap": 0.61748140276303,
37
+ "f1": 0.2709829606333994,
38
+ "hamming": 0.40621679064824656
39
+ },
40
+ {
41
+ "accuracy": 0.46014877789585545,
42
+ "lrap": 0.6721041445271096,
43
+ "f1": 0.3376165714261497,
44
+ "hamming": 0.47051009564293306
45
+ },
46
+ {
47
+ "accuracy": 0.4075451647183847,
48
+ "lrap": 0.6602019128586725,
49
+ "f1": 0.32479768240768186,
50
+ "hamming": 0.4094048884165781
51
+ },
52
+ {
53
+ "accuracy": 0.3416578108395324,
54
+ "lrap": 0.6032412327311492,
55
+ "f1": 0.21137090778126616,
56
+ "hamming": 0.3435175345377258
57
+ },
58
+ {
59
+ "accuracy": 0.3767268862911796,
60
+ "lrap": 0.6173219978746131,
61
+ "f1": 0.26288247765679607,
62
+ "hamming": 0.3793836344314559
63
+ },
64
+ {
65
+ "accuracy": 0.42454835281615305,
66
+ "lrap": 0.6846174282678109,
67
+ "f1": 0.3557173554948648,
68
+ "hamming": 0.4348211122918881
69
+ }
70
+ ],
71
+ "accuracy": 0.4139213602550478,
72
+ "lrap": 0.6548113708820515,
73
+ "f1": 0.30770806123991606,
74
+ "hamming": 0.41954481048529935,
75
+ "main_score": 0.4139213602550478,
76
+ "hf_subset": "default",
77
+ "languages": [
78
+ "rus-Cyrl"
79
+ ]
80
+ }
81
+ ]
82
+ },
83
+ "evaluation_time": 5.374143838882446,
84
+ "kg_co2_emissions": null,
85
+ "date": null
86
+ }
87
+ }
results/mteb_rus_v1_1/raw_task_results/013_SensitiveTopicsClassification.json ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "SensitiveTopicsClassification": {
3
+ "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2",
4
+ "task_name": "SensitiveTopicsClassification",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "scores_per_experiment": [
10
+ {
11
+ "accuracy": 0.3046875,
12
+ "lrap": 0.4618055555555483,
13
+ "f1": 0.34623624351305954,
14
+ "hamming": 0.35758463541666663
15
+ },
16
+ {
17
+ "accuracy": 0.3017578125,
18
+ "lrap": 0.4567871093749926,
19
+ "f1": 0.3448719517324777,
20
+ "hamming": 0.35689290364583337
21
+ },
22
+ {
23
+ "accuracy": 0.3056640625,
24
+ "lrap": 0.4673800998263812,
25
+ "f1": 0.3421298234205183,
26
+ "hamming": 0.3638102213541667
27
+ },
28
+ {
29
+ "accuracy": 0.2900390625,
30
+ "lrap": 0.4221598307291588,
31
+ "f1": 0.28561293212265776,
32
+ "hamming": 0.3297526041666667
33
+ },
34
+ {
35
+ "accuracy": 0.30615234375,
36
+ "lrap": 0.4629448784722148,
37
+ "f1": 0.33937458698387557,
38
+ "hamming": 0.36083984375
39
+ },
40
+ {
41
+ "accuracy": 0.26513671875,
42
+ "lrap": 0.4202270507812427,
43
+ "f1": 0.3003984789656055,
44
+ "hamming": 0.312353515625
45
+ },
46
+ {
47
+ "accuracy": 0.25830078125,
48
+ "lrap": 0.40121799045138135,
49
+ "f1": 0.27345999250866426,
50
+ "hamming": 0.3074951171875
51
+ },
52
+ {
53
+ "accuracy": 0.29296875,
54
+ "lrap": 0.46238878038193715,
55
+ "f1": 0.3779972935713676,
56
+ "hamming": 0.3636881510416667
57
+ },
58
+ {
59
+ "accuracy": 0.275390625,
60
+ "lrap": 0.4192572699652702,
61
+ "f1": 0.29515464665742086,
62
+ "hamming": 0.3192138671875
63
+ },
64
+ {
65
+ "accuracy": 0.26171875,
66
+ "lrap": 0.41849772135415897,
67
+ "f1": 0.28628125884105865,
68
+ "hamming": 0.31005859375
69
+ }
70
+ ],
71
+ "accuracy": 0.286181640625,
72
+ "lrap": 0.4392666286892286,
73
+ "f1": 0.31915172083167065,
74
+ "hamming": 0.3381689453125,
75
+ "main_score": 0.286181640625,
76
+ "hf_subset": "default",
77
+ "languages": [
78
+ "rus-Cyrl"
79
+ ]
80
+ }
81
+ ]
82
+ },
83
+ "evaluation_time": 20.475817680358887,
84
+ "kg_co2_emissions": null,
85
+ "date": null
86
+ }
87
+ }
results/mteb_rus_v1_1/raw_task_results/014_TERRa.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "TERRa": {
3
+ "dataset_revision": "48312c334eb916d88481dc4a41bd0d8b2fa0d576",
4
+ "task_name": "TERRa",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "dev": [
8
+ {
9
+ "similarity_accuracy": 0.5407166123778502,
10
+ "similarity_f1": 0.6727272727272727,
11
+ "similarity_precision": 0.5156794425087108,
12
+ "similarity_recall": 0.9673202614379085,
13
+ "similarity_ap": 0.5250119340487216,
14
+ "cosine_accuracy": 0.5407166123778502,
15
+ "cosine_f1": 0.6727272727272727,
16
+ "cosine_precision": 0.5156794425087108,
17
+ "cosine_recall": 0.9673202614379085,
18
+ "cosine_ap": 0.5250119340487216,
19
+ "manhattan_accuracy": 0.5309446254071661,
20
+ "manhattan_f1": 0.6711409395973154,
21
+ "manhattan_precision": 0.5102040816326531,
22
+ "manhattan_recall": 0.9803921568627451,
23
+ "manhattan_ap": 0.5219060049777211,
24
+ "euclidean_accuracy": 0.5407166123778502,
25
+ "euclidean_f1": 0.6727272727272727,
26
+ "euclidean_precision": 0.5156794425087108,
27
+ "euclidean_recall": 0.9673202614379085,
28
+ "euclidean_ap": 0.5250119340487216,
29
+ "dot_accuracy": 0.5407166123778502,
30
+ "dot_f1": 0.6727272727272727,
31
+ "dot_precision": 0.5156794425087108,
32
+ "dot_recall": 0.9673202614379085,
33
+ "dot_ap": 0.5250119340487216,
34
+ "max_accuracy": 0.5407166123778502,
35
+ "max_f1": 0.6727272727272727,
36
+ "max_precision": 0.5156794425087108,
37
+ "max_recall": 0.9803921568627451,
38
+ "max_ap": 0.5250119340487216,
39
+ "main_score": 0.5250119340487216,
40
+ "hf_subset": "default",
41
+ "languages": [
42
+ "rus-Cyrl"
43
+ ]
44
+ }
45
+ ]
46
+ },
47
+ "evaluation_time": 0.1239175796508789,
48
+ "kg_co2_emissions": null,
49
+ "date": null
50
+ }
51
+ }
results/mteb_rus_v1_1/raw_task_results/015_MIRACLReranking.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "MIRACLReranking": {
3
+ "dataset_revision": "d11a14c74e8bd448cedab0c1d9a720040535f228",
4
+ "task_name": "MIRACLReranking",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "dev": [
8
+ {
9
+ "ndcg_at_1": 0.18845,
10
+ "ndcg_at_3": 0.22547,
11
+ "ndcg_at_5": 0.2524,
12
+ "ndcg_at_10": 0.29416,
13
+ "ndcg_at_20": 0.33633,
14
+ "ndcg_at_100": 0.3991,
15
+ "ndcg_at_1000": 0.3991,
16
+ "map_at_1": 0.11125,
17
+ "map_at_3": 0.17378,
18
+ "map_at_5": 0.19622,
19
+ "map_at_10": 0.22036,
20
+ "map_at_20": 0.23679,
21
+ "map_at_100": 0.25194,
22
+ "map_at_1000": 0.25194,
23
+ "recall_at_1": 0.11125,
24
+ "recall_at_3": 0.23618,
25
+ "recall_at_5": 0.3081,
26
+ "recall_at_10": 0.41681,
27
+ "recall_at_20": 0.54579,
28
+ "recall_at_100": 0.79952,
29
+ "recall_at_1000": 0.79952,
30
+ "accuracy": 0.11125,
31
+ "precision_at_1": 0.18845,
32
+ "precision_at_3": 0.14542,
33
+ "precision_at_5": 0.12221,
34
+ "precision_at_10": 0.09046,
35
+ "precision_at_20": 0.06219,
36
+ "precision_at_100": 0.01953,
37
+ "precision_at_1000": 0.00195,
38
+ "mrr_at_1": 0.1884522854851644,
39
+ "mrr_at_3": 0.2634322373696872,
40
+ "mrr_at_5": 0.2854851643945465,
41
+ "mrr_at_10": 0.30013142653479435,
42
+ "mrr_at_20": 0.30697897022149495,
43
+ "mrr_at_100": 0.31067148211836554,
44
+ "mrr_at_1000": 0.31067148211836554,
45
+ "nauc_ndcg_at_1_max": 0.02210143631252207,
46
+ "nauc_ndcg_at_1_std": -0.021426635390588082,
47
+ "nauc_ndcg_at_1_diff1": 0.19575305781271488,
48
+ "nauc_ndcg_at_3_max": 0.004961240039109513,
49
+ "nauc_ndcg_at_3_std": -0.019017997354727863,
50
+ "nauc_ndcg_at_3_diff1": 0.17832390306430515,
51
+ "nauc_ndcg_at_5_max": 0.006208685812833753,
52
+ "nauc_ndcg_at_5_std": -0.011166001697484822,
53
+ "nauc_ndcg_at_5_diff1": 0.16250012027353145,
54
+ "nauc_ndcg_at_10_max": 0.017863430773509277,
55
+ "nauc_ndcg_at_10_std": 0.010480696045206237,
56
+ "nauc_ndcg_at_10_diff1": 0.14549085616492066,
57
+ "nauc_ndcg_at_20_max": 0.051451426876839935,
58
+ "nauc_ndcg_at_20_std": 0.021273804049843328,
59
+ "nauc_ndcg_at_20_diff1": 0.126970138513185,
60
+ "nauc_ndcg_at_100_max": 0.0878549273503891,
61
+ "nauc_ndcg_at_100_std": 0.013966016788884426,
62
+ "nauc_ndcg_at_100_diff1": 0.09177322700820684,
63
+ "nauc_ndcg_at_1000_max": 0.0878549273503891,
64
+ "nauc_ndcg_at_1000_std": 0.013966016788884426,
65
+ "nauc_ndcg_at_1000_diff1": 0.09177322700820684,
66
+ "nauc_map_at_1_max": -0.03672149258853317,
67
+ "nauc_map_at_1_std": -0.03530853836862883,
68
+ "nauc_map_at_1_diff1": 0.2783953716442577,
69
+ "nauc_map_at_3_max": -0.027278124734670617,
70
+ "nauc_map_at_3_std": -0.02310875021598933,
71
+ "nauc_map_at_3_diff1": 0.22100035771083082,
72
+ "nauc_map_at_5_max": -0.013653220717721149,
73
+ "nauc_map_at_5_std": -0.015768389466933118,
74
+ "nauc_map_at_5_diff1": 0.20093862530510687,
75
+ "nauc_map_at_10_max": -0.0029969905053822106,
76
+ "nauc_map_at_10_std": -0.004149926239420884,
77
+ "nauc_map_at_10_diff1": 0.1867557026624019,
78
+ "nauc_map_at_20_max": 0.010480979003614289,
79
+ "nauc_map_at_20_std": 0.002303769719954406,
80
+ "nauc_map_at_20_diff1": 0.1767835536788562,
81
+ "nauc_map_at_100_max": 0.01931102490572577,
82
+ "nauc_map_at_100_std": 0.0009574246172918966,
83
+ "nauc_map_at_100_diff1": 0.16893154683531492,
84
+ "nauc_map_at_1000_max": 0.01931102490572577,
85
+ "nauc_map_at_1000_std": 0.0009574246172918966,
86
+ "nauc_map_at_1000_diff1": 0.16893154683531492,
87
+ "nauc_recall_at_1_max": -0.03672149258853317,
88
+ "nauc_recall_at_1_std": -0.03530853836862883,
89
+ "nauc_recall_at_1_diff1": 0.2783953716442577,
90
+ "nauc_recall_at_3_max": -0.02647939750289965,
91
+ "nauc_recall_at_3_std": -0.014575435379617196,
92
+ "nauc_recall_at_3_diff1": 0.16216475085475326,
93
+ "nauc_recall_at_5_max": -0.010664974497512953,
94
+ "nauc_recall_at_5_std": 0.006933716874718121,
95
+ "nauc_recall_at_5_diff1": 0.12203048297099292,
96
+ "nauc_recall_at_10_max": 0.016217543267807236,
97
+ "nauc_recall_at_10_std": 0.04932862617247996,
98
+ "nauc_recall_at_10_diff1": 0.07842191878454743,
99
+ "nauc_recall_at_20_max": 0.0927279423341992,
100
+ "nauc_recall_at_20_std": 0.07724608826654757,
101
+ "nauc_recall_at_20_diff1": 0.02775842399529999,
102
+ "nauc_recall_at_100_max": 0.3397421993986906,
103
+ "nauc_recall_at_100_std": 0.07881238826787465,
104
+ "nauc_recall_at_100_diff1": -0.20194259535976652,
105
+ "nauc_recall_at_1000_max": 0.3397421993986906,
106
+ "nauc_recall_at_1000_std": 0.07881238826787465,
107
+ "nauc_recall_at_1000_diff1": -0.20194259535976652,
108
+ "nauc_precision_at_1_max": 0.02210143631252207,
109
+ "nauc_precision_at_1_std": -0.021426635390588082,
110
+ "nauc_precision_at_1_diff1": 0.19575305781271488,
111
+ "nauc_precision_at_3_max": 0.06232353010344895,
112
+ "nauc_precision_at_3_std": -0.0009646054891941732,
113
+ "nauc_precision_at_3_diff1": 0.10289216835292356,
114
+ "nauc_precision_at_5_max": 0.09384529362440312,
115
+ "nauc_precision_at_5_std": 0.008538101109706897,
116
+ "nauc_precision_at_5_diff1": 0.04295028893962814,
117
+ "nauc_precision_at_10_max": 0.10811288096779292,
118
+ "nauc_precision_at_10_std": 0.036836899408663415,
119
+ "nauc_precision_at_10_diff1": -0.008039192233450471,
120
+ "nauc_precision_at_20_max": 0.17064124277914539,
121
+ "nauc_precision_at_20_std": 0.05272296696406313,
122
+ "nauc_precision_at_20_diff1": -0.045165863661405035,
123
+ "nauc_precision_at_100_max": 0.19124420053008995,
124
+ "nauc_precision_at_100_std": 0.007591271418122008,
125
+ "nauc_precision_at_100_diff1": -0.10524129407275218,
126
+ "nauc_precision_at_1000_max": 0.19124420053008967,
127
+ "nauc_precision_at_1000_std": 0.007591271418121774,
128
+ "nauc_precision_at_1000_diff1": -0.10524129407275232,
129
+ "nauc_mrr_at_1_max": 0.02210143631252207,
130
+ "nauc_mrr_at_1_std": -0.021426635390588082,
131
+ "nauc_mrr_at_1_diff1": 0.19575305781271488,
132
+ "nauc_mrr_at_3_max": 0.03347543694339886,
133
+ "nauc_mrr_at_3_std": -0.018883204006479287,
134
+ "nauc_mrr_at_3_diff1": 0.15823852212846357,
135
+ "nauc_mrr_at_5_max": 0.03561557809649422,
136
+ "nauc_mrr_at_5_std": -0.017091085745613813,
137
+ "nauc_mrr_at_5_diff1": 0.1471576901124747,
138
+ "nauc_mrr_at_10_max": 0.040907694804838746,
139
+ "nauc_mrr_at_10_std": -0.009756234262662531,
140
+ "nauc_mrr_at_10_diff1": 0.14170344865262957,
141
+ "nauc_mrr_at_20_max": 0.047508694214849434,
142
+ "nauc_mrr_at_20_std": -0.009848386113933922,
143
+ "nauc_mrr_at_20_diff1": 0.14051998696030255,
144
+ "nauc_mrr_at_100_max": 0.04818428350930322,
145
+ "nauc_mrr_at_100_std": -0.009409925184704187,
146
+ "nauc_mrr_at_100_diff1": 0.13804705037076634,
147
+ "nauc_mrr_at_1000_max": 0.04818428350930322,
148
+ "nauc_mrr_at_1000_std": -0.009409925184704187,
149
+ "nauc_mrr_at_1000_diff1": 0.13804705037076634,
150
+ "hit_rate_at_1": 0.18845,
151
+ "hit_rate_at_3": 0.36247,
152
+ "hit_rate_at_5": 0.4587,
153
+ "hit_rate_at_10": 0.56616,
154
+ "hit_rate_at_20": 0.6656,
155
+ "hit_rate_at_100": 0.79952,
156
+ "hit_rate_at_1000": 0.79952,
157
+ "main_score": 0.29416,
158
+ "hf_subset": "ru",
159
+ "languages": [
160
+ "rus-Cyrl"
161
+ ]
162
+ }
163
+ ]
164
+ },
165
+ "evaluation_time": 30.278363704681396,
166
+ "kg_co2_emissions": null,
167
+ "date": null
168
+ }
169
+ }
results/mteb_rus_v1_1/raw_task_results/016_RuBQReranking.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuBQReranking": {
3
+ "dataset_revision": "e8233e2234f8b24ab47f203b69d1161c3c0bc5a1",
4
+ "task_name": "RuBQReranking",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "ndcg_at_1": 0.40232,
10
+ "ndcg_at_3": 0.51063,
11
+ "ndcg_at_5": 0.58086,
12
+ "ndcg_at_10": 0.64424,
13
+ "ndcg_at_20": 0.6712,
14
+ "ndcg_at_100": 0.67271,
15
+ "ndcg_at_1000": 0.67271,
16
+ "map_at_1": 0.2743,
17
+ "map_at_3": 0.43958,
18
+ "map_at_5": 0.49317,
19
+ "map_at_10": 0.53127,
20
+ "map_at_20": 0.54289,
21
+ "map_at_100": 0.54341,
22
+ "map_at_1000": 0.54341,
23
+ "recall_at_1": 0.2743,
24
+ "recall_at_3": 0.58561,
25
+ "recall_at_5": 0.74235,
26
+ "recall_at_10": 0.90511,
27
+ "recall_at_20": 0.99437,
28
+ "recall_at_100": 1.0,
29
+ "recall_at_1000": 1.0,
30
+ "accuracy": 0.2743,
31
+ "precision_at_1": 0.40232,
32
+ "precision_at_3": 0.30325,
33
+ "precision_at_5": 0.23894,
34
+ "precision_at_10": 0.15016,
35
+ "precision_at_20": 0.0833,
36
+ "precision_at_100": 0.01678,
37
+ "precision_at_1000": 0.00168,
38
+ "mrr_at_1": 0.4029658284977434,
39
+ "mrr_at_3": 0.549215559853859,
40
+ "mrr_at_5": 0.5764560498603075,
41
+ "mrr_at_10": 0.5902213625617895,
42
+ "mrr_at_20": 0.5937484462096584,
43
+ "mrr_at_100": 0.5938405526847436,
44
+ "mrr_at_1000": 0.5938405526847436,
45
+ "nauc_ndcg_at_1_max": -0.011119661366802385,
46
+ "nauc_ndcg_at_1_std": -0.05901104690423069,
47
+ "nauc_ndcg_at_1_diff1": 0.2164415608837586,
48
+ "nauc_ndcg_at_3_max": -0.034657403113067725,
49
+ "nauc_ndcg_at_3_std": -0.04431116979611783,
50
+ "nauc_ndcg_at_3_diff1": 0.15510185318393396,
51
+ "nauc_ndcg_at_5_max": -0.02578962706502498,
52
+ "nauc_ndcg_at_5_std": -0.0007439908997526085,
53
+ "nauc_ndcg_at_5_diff1": 0.15929913665655374,
54
+ "nauc_ndcg_at_10_max": -0.025600797580677948,
55
+ "nauc_ndcg_at_10_std": -0.0032482371523010357,
56
+ "nauc_ndcg_at_10_diff1": 0.1594831908847813,
57
+ "nauc_ndcg_at_20_max": -0.02016002181528052,
58
+ "nauc_ndcg_at_20_std": -0.0216838968030041,
59
+ "nauc_ndcg_at_20_diff1": 0.15832318653513078,
60
+ "nauc_ndcg_at_100_max": -0.020063718672623215,
61
+ "nauc_ndcg_at_100_std": -0.024851951509289316,
62
+ "nauc_ndcg_at_100_diff1": 0.1584612430764489,
63
+ "nauc_ndcg_at_1000_max": -0.020063718672623215,
64
+ "nauc_ndcg_at_1000_std": -0.024851951509289316,
65
+ "nauc_ndcg_at_1000_diff1": 0.1584612430764489,
66
+ "nauc_map_at_1_max": -0.021530047326232295,
67
+ "nauc_map_at_1_std": -0.07979670637647832,
68
+ "nauc_map_at_1_diff1": 0.21090331400254597,
69
+ "nauc_map_at_3_max": -0.02896362830947275,
70
+ "nauc_map_at_3_std": -0.05260852225989718,
71
+ "nauc_map_at_3_diff1": 0.17157328465186059,
72
+ "nauc_map_at_5_max": -0.02571661773109319,
73
+ "nauc_map_at_5_std": -0.01978878978219035,
74
+ "nauc_map_at_5_diff1": 0.16920077274581852,
75
+ "nauc_map_at_10_max": -0.025990406793149217,
76
+ "nauc_map_at_10_std": -0.018292375316861054,
77
+ "nauc_map_at_10_diff1": 0.1659684791167953,
78
+ "nauc_map_at_20_max": -0.022880580797277922,
79
+ "nauc_map_at_20_std": -0.024445043300775123,
80
+ "nauc_map_at_20_diff1": 0.16491677152036283,
81
+ "nauc_map_at_100_max": -0.022994358730534112,
82
+ "nauc_map_at_100_std": -0.025362356620525727,
83
+ "nauc_map_at_100_diff1": 0.16469647471688642,
84
+ "nauc_map_at_1000_max": -0.022994358730534112,
85
+ "nauc_map_at_1000_std": -0.025362356620525727,
86
+ "nauc_map_at_1000_diff1": 0.16469647471688642,
87
+ "nauc_recall_at_1_max": -0.021530047326232295,
88
+ "nauc_recall_at_1_std": -0.07979670637647832,
89
+ "nauc_recall_at_1_diff1": 0.21090331400254597,
90
+ "nauc_recall_at_3_max": -0.051874160500883615,
91
+ "nauc_recall_at_3_std": -0.039328480393320014,
92
+ "nauc_recall_at_3_diff1": 0.11563861099710837,
93
+ "nauc_recall_at_5_max": -0.03531536301378802,
94
+ "nauc_recall_at_5_std": 0.07716490783641676,
95
+ "nauc_recall_at_5_diff1": 0.11199012536545842,
96
+ "nauc_recall_at_10_max": -0.04847773370082636,
97
+ "nauc_recall_at_10_std": 0.15360120135052013,
98
+ "nauc_recall_at_10_diff1": 0.11795062291714448,
99
+ "nauc_recall_at_20_max": -0.06196162824323477,
100
+ "nauc_recall_at_20_std": 0.3828851380887754,
101
+ "nauc_recall_at_20_diff1": 0.029761429703057278,
102
+ "nauc_recall_at_100_max": NaN,
103
+ "nauc_recall_at_100_std": NaN,
104
+ "nauc_recall_at_100_diff1": NaN,
105
+ "nauc_recall_at_1000_max": NaN,
106
+ "nauc_recall_at_1000_std": NaN,
107
+ "nauc_recall_at_1000_diff1": NaN,
108
+ "nauc_precision_at_1_max": -0.011119661366802385,
109
+ "nauc_precision_at_1_std": -0.05901104690423069,
110
+ "nauc_precision_at_1_diff1": 0.2164415608837586,
111
+ "nauc_precision_at_3_max": -0.006328276318049836,
112
+ "nauc_precision_at_3_std": 0.030549412096263207,
113
+ "nauc_precision_at_3_diff1": 0.04905441806975031,
114
+ "nauc_precision_at_5_max": 0.010425798799772568,
115
+ "nauc_precision_at_5_std": 0.11144839884342704,
116
+ "nauc_precision_at_5_diff1": -0.006651057083024945,
117
+ "nauc_precision_at_10_max": 0.01803728226481624,
118
+ "nauc_precision_at_10_std": 0.10776607647322455,
119
+ "nauc_precision_at_10_diff1": -0.06965952665447751,
120
+ "nauc_precision_at_20_max": 0.03562909978457904,
121
+ "nauc_precision_at_20_std": 0.07076258080298485,
122
+ "nauc_precision_at_20_diff1": -0.09932505395223064,
123
+ "nauc_precision_at_100_max": 0.03475836319483572,
124
+ "nauc_precision_at_100_std": 0.06295728068877972,
125
+ "nauc_precision_at_100_diff1": -0.10232290414272635,
126
+ "nauc_precision_at_1000_max": 0.03475836319483504,
127
+ "nauc_precision_at_1000_std": 0.06295728068877915,
128
+ "nauc_precision_at_1000_diff1": -0.10232290414272718,
129
+ "nauc_mrr_at_1_max": -0.010816854076996025,
130
+ "nauc_mrr_at_1_std": -0.056972801133246424,
131
+ "nauc_mrr_at_1_diff1": 0.2147091864358949,
132
+ "nauc_mrr_at_3_max": -0.02212773196875393,
133
+ "nauc_mrr_at_3_std": -0.043432799434509776,
134
+ "nauc_mrr_at_3_diff1": 0.16768318319120454,
135
+ "nauc_mrr_at_5_max": -0.017296530048787614,
136
+ "nauc_mrr_at_5_std": -0.03375591606772892,
137
+ "nauc_mrr_at_5_diff1": 0.17106367877308443,
138
+ "nauc_mrr_at_10_max": -0.0168633186543582,
139
+ "nauc_mrr_at_10_std": -0.038202695399556286,
140
+ "nauc_mrr_at_10_diff1": 0.17450758379514633,
141
+ "nauc_mrr_at_20_max": -0.017509749719579016,
142
+ "nauc_mrr_at_20_std": -0.040765187209673946,
143
+ "nauc_mrr_at_20_diff1": 0.17523539068478278,
144
+ "nauc_mrr_at_100_max": -0.01742011159112179,
145
+ "nauc_mrr_at_100_std": -0.04084509016096958,
146
+ "nauc_mrr_at_100_diff1": 0.17543683754692133,
147
+ "nauc_mrr_at_1000_max": -0.01742011159112179,
148
+ "nauc_mrr_at_1000_std": -0.04084509016096958,
149
+ "nauc_mrr_at_1000_diff1": 0.17543683754692133,
150
+ "hit_rate_at_1": 0.40232,
151
+ "hit_rate_at_3": 0.73436,
152
+ "hit_rate_at_5": 0.85493,
153
+ "hit_rate_at_10": 0.95035,
154
+ "hit_rate_at_20": 0.99807,
155
+ "hit_rate_at_100": 1.0,
156
+ "hit_rate_at_1000": 1.0,
157
+ "main_score": 0.54341,
158
+ "hf_subset": "default",
159
+ "languages": [
160
+ "rus-Cyrl"
161
+ ]
162
+ }
163
+ ]
164
+ },
165
+ "evaluation_time": 8.462158679962158,
166
+ "kg_co2_emissions": null,
167
+ "date": null
168
+ }
169
+ }
results/mteb_rus_v1_1/raw_task_results/017_MIRACLRetrievalHardNegatives_v2.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "MIRACLRetrievalHardNegatives.v2": {
3
+ "dataset_revision": "d7d94fa4b946cec4a27c84653aa0cf6b33f74a3c",
4
+ "task_name": "MIRACLRetrievalHardNegatives.v2",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "dev": [
8
+ {
9
+ "ndcg_at_1": 0.147,
10
+ "ndcg_at_3": 0.16699,
11
+ "ndcg_at_5": 0.18976,
12
+ "ndcg_at_10": 0.23447,
13
+ "ndcg_at_20": 0.27987,
14
+ "ndcg_at_100": 0.36216,
15
+ "ndcg_at_1000": 0.39259,
16
+ "map_at_1": 0.0662,
17
+ "map_at_3": 0.1128,
18
+ "map_at_5": 0.13188,
19
+ "map_at_10": 0.15461,
20
+ "map_at_20": 0.17067,
21
+ "map_at_100": 0.18852,
22
+ "map_at_1000": 0.19094,
23
+ "recall_at_1": 0.0662,
24
+ "recall_at_3": 0.1637,
25
+ "recall_at_5": 0.22819,
26
+ "recall_at_10": 0.34811,
27
+ "recall_at_20": 0.48166,
28
+ "recall_at_100": 0.80746,
29
+ "recall_at_1000": 0.97862,
30
+ "accuracy": 0.0662,
31
+ "precision_at_1": 0.147,
32
+ "precision_at_3": 0.12367,
33
+ "precision_at_5": 0.1086,
34
+ "precision_at_10": 0.0861,
35
+ "precision_at_20": 0.0625,
36
+ "precision_at_100": 0.02266,
37
+ "precision_at_1000": 0.00278,
38
+ "mrr_at_1": 0.147,
39
+ "mrr_at_3": 0.21983333333333357,
40
+ "mrr_at_5": 0.2448833333333334,
41
+ "mrr_at_10": 0.2650015873015871,
42
+ "mrr_at_20": 0.27426807402693604,
43
+ "mrr_at_100": 0.27985727245075415,
44
+ "mrr_at_1000": 0.2802840751730957,
45
+ "nauc_ndcg_at_1_max": 0.01963167836217812,
46
+ "nauc_ndcg_at_1_std": -0.03857165368188621,
47
+ "nauc_ndcg_at_1_diff1": 0.11073592701509673,
48
+ "nauc_ndcg_at_3_max": -0.014087543796952288,
49
+ "nauc_ndcg_at_3_std": -0.06997807904625505,
50
+ "nauc_ndcg_at_3_diff1": 0.11170377583193272,
51
+ "nauc_ndcg_at_5_max": -0.0014355042767106145,
52
+ "nauc_ndcg_at_5_std": -0.06320214660793566,
53
+ "nauc_ndcg_at_5_diff1": 0.11578879906190812,
54
+ "nauc_ndcg_at_10_max": -0.004782406326853926,
55
+ "nauc_ndcg_at_10_std": -0.08472432117515467,
56
+ "nauc_ndcg_at_10_diff1": 0.11201079896264193,
57
+ "nauc_ndcg_at_20_max": 0.01748314785742713,
58
+ "nauc_ndcg_at_20_std": -0.07633601638687278,
59
+ "nauc_ndcg_at_20_diff1": 0.11321204994621799,
60
+ "nauc_ndcg_at_100_max": 0.04069119285510586,
61
+ "nauc_ndcg_at_100_std": -0.037275076636547506,
62
+ "nauc_ndcg_at_100_diff1": 0.11881761465338446,
63
+ "nauc_ndcg_at_1000_max": 0.033993602928395636,
64
+ "nauc_ndcg_at_1000_std": -0.03437047998166715,
65
+ "nauc_ndcg_at_1000_diff1": 0.11221901455913316,
66
+ "nauc_map_at_1_max": 0.0056513272282884075,
67
+ "nauc_map_at_1_std": -0.04709406381292279,
68
+ "nauc_map_at_1_diff1": 0.15255137564624924,
69
+ "nauc_map_at_3_max": -0.027111322474933697,
70
+ "nauc_map_at_3_std": -0.08072320690770424,
71
+ "nauc_map_at_3_diff1": 0.127254854432919,
72
+ "nauc_map_at_5_max": -0.009596794702362181,
73
+ "nauc_map_at_5_std": -0.07344615888680858,
74
+ "nauc_map_at_5_diff1": 0.12609277172319572,
75
+ "nauc_map_at_10_max": -0.011574940837646886,
76
+ "nauc_map_at_10_std": -0.0832190661520011,
77
+ "nauc_map_at_10_diff1": 0.12553032270013903,
78
+ "nauc_map_at_20_max": -0.0016962203105660345,
79
+ "nauc_map_at_20_std": -0.07813541848111363,
80
+ "nauc_map_at_20_diff1": 0.12483696202400532,
81
+ "nauc_map_at_100_max": 0.004824007953821765,
82
+ "nauc_map_at_100_std": -0.06893788294054101,
83
+ "nauc_map_at_100_diff1": 0.12480880227253366,
84
+ "nauc_map_at_1000_max": 0.004984699686724829,
85
+ "nauc_map_at_1000_std": -0.0670796756719255,
86
+ "nauc_map_at_1000_diff1": 0.12422861898365495,
87
+ "nauc_recall_at_1_max": 0.0056513272282884075,
88
+ "nauc_recall_at_1_std": -0.04709406381292279,
89
+ "nauc_recall_at_1_diff1": 0.15255137564624924,
90
+ "nauc_recall_at_3_max": -0.04023153608259714,
91
+ "nauc_recall_at_3_std": -0.0788368147808212,
92
+ "nauc_recall_at_3_diff1": 0.10630219403732986,
93
+ "nauc_recall_at_5_max": -0.007067432941490116,
94
+ "nauc_recall_at_5_std": -0.06094495194099402,
95
+ "nauc_recall_at_5_diff1": 0.10112364726561245,
96
+ "nauc_recall_at_10_max": -0.00906296693681506,
97
+ "nauc_recall_at_10_std": -0.08997255626644053,
98
+ "nauc_recall_at_10_diff1": 0.08348174881184926,
99
+ "nauc_recall_at_20_max": 0.03697709731114241,
100
+ "nauc_recall_at_20_std": -0.0788782456024569,
101
+ "nauc_recall_at_20_diff1": 0.08711734918740577,
102
+ "nauc_recall_at_100_max": 0.14473399229842365,
103
+ "nauc_recall_at_100_std": 0.06968671289737048,
104
+ "nauc_recall_at_100_diff1": 0.12196764045202206,
105
+ "nauc_recall_at_1000_max": 0.5518596125213805,
106
+ "nauc_recall_at_1000_std": 0.5043326495532389,
107
+ "nauc_recall_at_1000_diff1": 0.1156961138578814,
108
+ "nauc_precision_at_1_max": 0.01963167836217812,
109
+ "nauc_precision_at_1_std": -0.03857165368188621,
110
+ "nauc_precision_at_1_diff1": 0.11073592701509673,
111
+ "nauc_precision_at_3_max": -0.002120362172739014,
112
+ "nauc_precision_at_3_std": -0.06178703486215531,
113
+ "nauc_precision_at_3_diff1": 0.09489125571142651,
114
+ "nauc_precision_at_5_max": 0.027377193268010032,
115
+ "nauc_precision_at_5_std": -0.040368401415310645,
116
+ "nauc_precision_at_5_diff1": 0.08583345645098964,
117
+ "nauc_precision_at_10_max": 0.02256641927503491,
118
+ "nauc_precision_at_10_std": -0.05155764532337613,
119
+ "nauc_precision_at_10_diff1": 0.06809016733508508,
120
+ "nauc_precision_at_20_max": 0.0735771712863975,
121
+ "nauc_precision_at_20_std": -0.001542072947008616,
122
+ "nauc_precision_at_20_diff1": 0.049057286866443804,
123
+ "nauc_precision_at_100_max": 0.09374163144482736,
124
+ "nauc_precision_at_100_std": 0.08133552800736653,
125
+ "nauc_precision_at_100_diff1": 0.027321556895815325,
126
+ "nauc_precision_at_1000_max": 0.06433593589975965,
127
+ "nauc_precision_at_1000_std": 0.0898064994761385,
128
+ "nauc_precision_at_1000_diff1": -0.004505154158990202,
129
+ "nauc_mrr_at_1_max": 0.01963167836217812,
130
+ "nauc_mrr_at_1_std": -0.03857165368188621,
131
+ "nauc_mrr_at_1_diff1": 0.11073592701509673,
132
+ "nauc_mrr_at_3_max": 0.0007188983434189348,
133
+ "nauc_mrr_at_3_std": -0.059457964982924764,
134
+ "nauc_mrr_at_3_diff1": 0.10153970493743683,
135
+ "nauc_mrr_at_5_max": 0.0046356605452382215,
136
+ "nauc_mrr_at_5_std": -0.057556176937728645,
137
+ "nauc_mrr_at_5_diff1": 0.10151388465633707,
138
+ "nauc_mrr_at_10_max": 0.007370451848529925,
139
+ "nauc_mrr_at_10_std": -0.06176199002452351,
140
+ "nauc_mrr_at_10_diff1": 0.09981755841693224,
141
+ "nauc_mrr_at_20_max": 0.009687394302992812,
142
+ "nauc_mrr_at_20_std": -0.06151630784177807,
143
+ "nauc_mrr_at_20_diff1": 0.10058920327184681,
144
+ "nauc_mrr_at_100_max": 0.010892510418175527,
145
+ "nauc_mrr_at_100_std": -0.059001444963431356,
146
+ "nauc_mrr_at_100_diff1": 0.10142939103206046,
147
+ "nauc_mrr_at_1000_max": 0.010719164891059028,
148
+ "nauc_mrr_at_1000_std": -0.059107551208974346,
149
+ "nauc_mrr_at_1000_diff1": 0.10137115846506142,
150
+ "hit_rate_at_1": 0.147,
151
+ "hit_rate_at_3": 0.319,
152
+ "hit_rate_at_5": 0.428,
153
+ "hit_rate_at_10": 0.578,
154
+ "hit_rate_at_20": 0.706,
155
+ "hit_rate_at_100": 0.915,
156
+ "hit_rate_at_1000": 0.992,
157
+ "main_score": 0.23447,
158
+ "hf_subset": "ru",
159
+ "languages": [
160
+ "rus-Cyrl"
161
+ ]
162
+ }
163
+ ]
164
+ },
165
+ "evaluation_time": 52.46664357185364,
166
+ "kg_co2_emissions": null,
167
+ "date": null
168
+ }
169
+ }
results/mteb_rus_v1_1/raw_task_results/018_RiaNewsRetrievalHardNegatives_v2.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RiaNewsRetrievalHardNegatives.v2": {
3
+ "dataset_revision": "d42860a6c15f0a2c4485bda10c6e5b641fdfe479",
4
+ "task_name": "RiaNewsRetrievalHardNegatives.v2",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "ndcg_at_1": 0.368,
10
+ "ndcg_at_3": 0.45664,
11
+ "ndcg_at_5": 0.48853,
12
+ "ndcg_at_10": 0.51899,
13
+ "ndcg_at_20": 0.53992,
14
+ "ndcg_at_100": 0.56355,
15
+ "ndcg_at_1000": 0.57394,
16
+ "map_at_1": 0.368,
17
+ "map_at_3": 0.43567,
18
+ "map_at_5": 0.45347,
19
+ "map_at_10": 0.46626,
20
+ "map_at_20": 0.47197,
21
+ "map_at_100": 0.47525,
22
+ "map_at_1000": 0.47568,
23
+ "recall_at_1": 0.368,
24
+ "recall_at_3": 0.517,
25
+ "recall_at_5": 0.594,
26
+ "recall_at_10": 0.687,
27
+ "recall_at_20": 0.77,
28
+ "recall_at_100": 0.897,
29
+ "recall_at_1000": 0.977,
30
+ "accuracy": 0.368,
31
+ "precision_at_1": 0.368,
32
+ "precision_at_3": 0.17233,
33
+ "precision_at_5": 0.1188,
34
+ "precision_at_10": 0.0687,
35
+ "precision_at_20": 0.0385,
36
+ "precision_at_100": 0.00897,
37
+ "precision_at_1000": 0.00098,
38
+ "mrr_at_1": 0.369,
39
+ "mrr_at_3": 0.43633333333333313,
40
+ "mrr_at_5": 0.45413333333333283,
41
+ "mrr_at_10": 0.4669369047619045,
42
+ "mrr_at_20": 0.4726485893277684,
43
+ "mrr_at_100": 0.4759311613771692,
44
+ "mrr_at_1000": 0.47635210201565925,
45
+ "nauc_ndcg_at_1_max": 0.0030180797222687473,
46
+ "nauc_ndcg_at_1_std": -0.10119782081311704,
47
+ "nauc_ndcg_at_1_diff1": 0.5114841266521666,
48
+ "nauc_ndcg_at_3_max": -0.024691184814318867,
49
+ "nauc_ndcg_at_3_std": -0.10407919091008486,
50
+ "nauc_ndcg_at_3_diff1": 0.44680578247500613,
51
+ "nauc_ndcg_at_5_max": -0.025289525593912957,
52
+ "nauc_ndcg_at_5_std": -0.10939382225310798,
53
+ "nauc_ndcg_at_5_diff1": 0.43333647983812956,
54
+ "nauc_ndcg_at_10_max": -0.023045116644416844,
55
+ "nauc_ndcg_at_10_std": -0.10583033521355914,
56
+ "nauc_ndcg_at_10_diff1": 0.425414502661872,
57
+ "nauc_ndcg_at_20_max": -0.0233049620535428,
58
+ "nauc_ndcg_at_20_std": -0.09843711196758224,
59
+ "nauc_ndcg_at_20_diff1": 0.43066147864434456,
60
+ "nauc_ndcg_at_100_max": -0.015414753442876546,
61
+ "nauc_ndcg_at_100_std": -0.09397548007141716,
62
+ "nauc_ndcg_at_100_diff1": 0.43517589759998526,
63
+ "nauc_ndcg_at_1000_max": -0.0129393753561436,
64
+ "nauc_ndcg_at_1000_std": -0.09275570042313806,
65
+ "nauc_ndcg_at_1000_diff1": 0.4393275420555011,
66
+ "nauc_map_at_1_max": 0.0030180797222687473,
67
+ "nauc_map_at_1_std": -0.10119782081311704,
68
+ "nauc_map_at_1_diff1": 0.5114841266521666,
69
+ "nauc_map_at_3_max": -0.018051417945429474,
70
+ "nauc_map_at_3_std": -0.10519003945744798,
71
+ "nauc_map_at_3_diff1": 0.46292232524583016,
72
+ "nauc_map_at_5_max": -0.018368020203616393,
73
+ "nauc_map_at_5_std": -0.10809470643149749,
74
+ "nauc_map_at_5_diff1": 0.45575577014138907,
75
+ "nauc_map_at_10_max": -0.01693978573888253,
76
+ "nauc_map_at_10_std": -0.10680484672460977,
77
+ "nauc_map_at_10_diff1": 0.4532282766171296,
78
+ "nauc_map_at_20_max": -0.017016106471545892,
79
+ "nauc_map_at_20_std": -0.10465093787054296,
80
+ "nauc_map_at_20_diff1": 0.45501328638401434,
81
+ "nauc_map_at_100_max": -0.01611401884215045,
82
+ "nauc_map_at_100_std": -0.10415291141498237,
83
+ "nauc_map_at_100_diff1": 0.4558849372360667,
84
+ "nauc_map_at_1000_max": -0.015989371494792685,
85
+ "nauc_map_at_1000_std": -0.1040095078750231,
86
+ "nauc_map_at_1000_diff1": 0.45606307945174707,
87
+ "nauc_recall_at_1_max": 0.0030180797222687473,
88
+ "nauc_recall_at_1_std": -0.10119782081311704,
89
+ "nauc_recall_at_1_diff1": 0.5114841266521666,
90
+ "nauc_recall_at_3_max": -0.04423691557926462,
91
+ "nauc_recall_at_3_std": -0.10003240849346617,
92
+ "nauc_recall_at_3_diff1": 0.3990963589938995,
93
+ "nauc_recall_at_5_max": -0.04697804546566737,
94
+ "nauc_recall_at_5_std": -0.11350497197826653,
95
+ "nauc_recall_at_5_diff1": 0.3606018260559373,
96
+ "nauc_recall_at_10_max": -0.045257838750217286,
97
+ "nauc_recall_at_10_std": -0.09995250038998808,
98
+ "nauc_recall_at_10_diff1": 0.31634389717649025,
99
+ "nauc_recall_at_20_max": -0.051854255723909566,
100
+ "nauc_recall_at_20_std": -0.05886394664398722,
101
+ "nauc_recall_at_20_diff1": 0.3151451399923902,
102
+ "nauc_recall_at_100_max": 0.04219104037563845,
103
+ "nauc_recall_at_100_std": 0.050844912778793834,
104
+ "nauc_recall_at_100_diff1": 0.25727298372052876,
105
+ "nauc_recall_at_1000_max": 0.42956602931027565,
106
+ "nauc_recall_at_1000_std": 0.5831405025778416,
107
+ "nauc_recall_at_1000_diff1": 0.017395363942680465,
108
+ "nauc_precision_at_1_max": 0.0030180797222687473,
109
+ "nauc_precision_at_1_std": -0.10119782081311704,
110
+ "nauc_precision_at_1_diff1": 0.5114841266521666,
111
+ "nauc_precision_at_3_max": -0.04423691557926432,
112
+ "nauc_precision_at_3_std": -0.10003240849346624,
113
+ "nauc_precision_at_3_diff1": 0.39909635899389956,
114
+ "nauc_precision_at_5_max": -0.04697804546566804,
115
+ "nauc_precision_at_5_std": -0.11350497197826696,
116
+ "nauc_precision_at_5_diff1": 0.36060182605593694,
117
+ "nauc_precision_at_10_max": -0.04525783875021779,
118
+ "nauc_precision_at_10_std": -0.09995250038998814,
119
+ "nauc_precision_at_10_diff1": 0.3163438971764898,
120
+ "nauc_precision_at_20_max": -0.051854255723911,
121
+ "nauc_precision_at_20_std": -0.058863946643988974,
122
+ "nauc_precision_at_20_diff1": 0.3151451399923896,
123
+ "nauc_precision_at_100_max": 0.042191040375639574,
124
+ "nauc_precision_at_100_std": 0.050844912778795125,
125
+ "nauc_precision_at_100_diff1": 0.2572729837205294,
126
+ "nauc_precision_at_1000_max": 0.42956602931026816,
127
+ "nauc_precision_at_1000_std": 0.583140502577831,
128
+ "nauc_precision_at_1000_diff1": 0.017395363942671063,
129
+ "nauc_mrr_at_1_max": 0.0005839908129230644,
130
+ "nauc_mrr_at_1_std": -0.0995723933040945,
131
+ "nauc_mrr_at_1_diff1": 0.5084404309773811,
132
+ "nauc_mrr_at_3_max": -0.019543966747207233,
133
+ "nauc_mrr_at_3_std": -0.10428977812330217,
134
+ "nauc_mrr_at_3_diff1": 0.4612479956887985,
135
+ "nauc_mrr_at_5_max": -0.019888009971783964,
136
+ "nauc_mrr_at_5_std": -0.10717731001701278,
137
+ "nauc_mrr_at_5_diff1": 0.4540519414127107,
138
+ "nauc_mrr_at_10_max": -0.01847282155469821,
139
+ "nauc_mrr_at_10_std": -0.10583322303725243,
140
+ "nauc_mrr_at_10_diff1": 0.45149446887661215,
141
+ "nauc_mrr_at_20_max": -0.018567103772982863,
142
+ "nauc_mrr_at_20_std": -0.10366830137425742,
143
+ "nauc_mrr_at_20_diff1": 0.4532588351147489,
144
+ "nauc_mrr_at_100_max": -0.01768067167972192,
145
+ "nauc_mrr_at_100_std": -0.1031612760642376,
146
+ "nauc_mrr_at_100_diff1": 0.4541171274943535,
147
+ "nauc_mrr_at_1000_max": -0.017558015703403276,
148
+ "nauc_mrr_at_1000_std": -0.10301665211124415,
149
+ "nauc_mrr_at_1000_diff1": 0.4542930158937033,
150
+ "hit_rate_at_1": 0.368,
151
+ "hit_rate_at_3": 0.517,
152
+ "hit_rate_at_5": 0.594,
153
+ "hit_rate_at_10": 0.687,
154
+ "hit_rate_at_20": 0.77,
155
+ "hit_rate_at_100": 0.897,
156
+ "hit_rate_at_1000": 0.977,
157
+ "main_score": 0.51899,
158
+ "hf_subset": "default",
159
+ "languages": [
160
+ "rus-Cyrl"
161
+ ]
162
+ }
163
+ ]
164
+ },
165
+ "evaluation_time": 93.29252910614014,
166
+ "kg_co2_emissions": null,
167
+ "date": null
168
+ }
169
+ }
results/mteb_rus_v1_1/raw_task_results/019_RuBQRetrieval.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuBQRetrieval": {
3
+ "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b",
4
+ "task_name": "RuBQRetrieval",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "ndcg_at_1": 0.26182,
10
+ "ndcg_at_3": 0.32482,
11
+ "ndcg_at_5": 0.36889,
12
+ "ndcg_at_10": 0.41913,
13
+ "ndcg_at_20": 0.45095,
14
+ "ndcg_at_100": 0.4843,
15
+ "ndcg_at_1000": 0.49829,
16
+ "map_at_1": 0.17347,
17
+ "map_at_3": 0.27139,
18
+ "map_at_5": 0.30096,
19
+ "map_at_10": 0.32615,
20
+ "map_at_20": 0.33762,
21
+ "map_at_100": 0.34378,
22
+ "map_at_1000": 0.34448,
23
+ "recall_at_1": 0.17347,
24
+ "recall_at_3": 0.37257,
25
+ "recall_at_5": 0.47456,
26
+ "recall_at_10": 0.60955,
27
+ "recall_at_20": 0.71684,
28
+ "recall_at_100": 0.87339,
29
+ "recall_at_1000": 0.96872,
30
+ "accuracy": 0.17347,
31
+ "precision_at_1": 0.26182,
32
+ "precision_at_3": 0.19405,
33
+ "precision_at_5": 0.15047,
34
+ "precision_at_10": 0.09905,
35
+ "precision_at_20": 0.05928,
36
+ "precision_at_100": 0.01452,
37
+ "precision_at_1000": 0.00162,
38
+ "mrr_at_1": 0.2624113475177305,
39
+ "mrr_at_3": 0.36623325453112676,
40
+ "mrr_at_5": 0.3906126871552408,
41
+ "mrr_at_10": 0.4065030582761082,
42
+ "mrr_at_20": 0.4124911714487776,
43
+ "mrr_at_100": 0.4154054136392914,
44
+ "mrr_at_1000": 0.4156870510182511,
45
+ "nauc_ndcg_at_1_max": 0.07717968432014255,
46
+ "nauc_ndcg_at_1_std": -0.12128970836264957,
47
+ "nauc_ndcg_at_1_diff1": 0.2376849423041946,
48
+ "nauc_ndcg_at_3_max": 0.11413697718620407,
49
+ "nauc_ndcg_at_3_std": -0.10646912135450516,
50
+ "nauc_ndcg_at_3_diff1": 0.19651143173782998,
51
+ "nauc_ndcg_at_5_max": 0.13088338374829245,
52
+ "nauc_ndcg_at_5_std": -0.11394560826582908,
53
+ "nauc_ndcg_at_5_diff1": 0.1967826846247824,
54
+ "nauc_ndcg_at_10_max": 0.1419244501498779,
55
+ "nauc_ndcg_at_10_std": -0.10498446411456383,
56
+ "nauc_ndcg_at_10_diff1": 0.19077840576521102,
57
+ "nauc_ndcg_at_20_max": 0.14642373135645612,
58
+ "nauc_ndcg_at_20_std": -0.09424288097592244,
59
+ "nauc_ndcg_at_20_diff1": 0.18757961730899603,
60
+ "nauc_ndcg_at_100_max": 0.1514760374986726,
61
+ "nauc_ndcg_at_100_std": -0.07621327482020097,
62
+ "nauc_ndcg_at_100_diff1": 0.19154116675370988,
63
+ "nauc_ndcg_at_1000_max": 0.1447507898909371,
64
+ "nauc_ndcg_at_1000_std": -0.08305725345200285,
65
+ "nauc_ndcg_at_1000_diff1": 0.19352463848232093,
66
+ "nauc_map_at_1_max": 0.07845970918553082,
67
+ "nauc_map_at_1_std": -0.10784255345594926,
68
+ "nauc_map_at_1_diff1": 0.22361824784646575,
69
+ "nauc_map_at_3_max": 0.11003922441924663,
70
+ "nauc_map_at_3_std": -0.10781868225048467,
71
+ "nauc_map_at_3_diff1": 0.20248553388819102,
72
+ "nauc_map_at_5_max": 0.11988884058961209,
73
+ "nauc_map_at_5_std": -0.11261335981615468,
74
+ "nauc_map_at_5_diff1": 0.2003170301276694,
75
+ "nauc_map_at_10_max": 0.12628393411057953,
76
+ "nauc_map_at_10_std": -0.10745184108059593,
77
+ "nauc_map_at_10_diff1": 0.19755096494279223,
78
+ "nauc_map_at_20_max": 0.1271236072865903,
79
+ "nauc_map_at_20_std": -0.1047192681936202,
80
+ "nauc_map_at_20_diff1": 0.19661711015908662,
81
+ "nauc_map_at_100_max": 0.12879058778701516,
82
+ "nauc_map_at_100_std": -0.10074105913536416,
83
+ "nauc_map_at_100_diff1": 0.19659442400229843,
84
+ "nauc_map_at_1000_max": 0.12854964851004685,
85
+ "nauc_map_at_1000_std": -0.10094663156136165,
86
+ "nauc_map_at_1000_diff1": 0.19672635905549946,
87
+ "nauc_recall_at_1_max": 0.07845970918553082,
88
+ "nauc_recall_at_1_std": -0.10784255345594926,
89
+ "nauc_recall_at_1_diff1": 0.22361824784646575,
90
+ "nauc_recall_at_3_max": 0.11869816314993642,
91
+ "nauc_recall_at_3_std": -0.0910871458296447,
92
+ "nauc_recall_at_3_diff1": 0.16049603280308827,
93
+ "nauc_recall_at_5_max": 0.13808350593932678,
94
+ "nauc_recall_at_5_std": -0.10594700276211821,
95
+ "nauc_recall_at_5_diff1": 0.15150543155311344,
96
+ "nauc_recall_at_10_max": 0.16412177929424385,
97
+ "nauc_recall_at_10_std": -0.08014742681452243,
98
+ "nauc_recall_at_10_diff1": 0.12823594149912823,
99
+ "nauc_recall_at_20_max": 0.19196380429449872,
100
+ "nauc_recall_at_20_std": -0.038351033488249385,
101
+ "nauc_recall_at_20_diff1": 0.10813504720431581,
102
+ "nauc_recall_at_100_max": 0.29166281916940984,
103
+ "nauc_recall_at_100_std": 0.16494006212211243,
104
+ "nauc_recall_at_100_diff1": 0.10786264144921465,
105
+ "nauc_recall_at_1000_max": 0.4194493510868826,
106
+ "nauc_recall_at_1000_std": 0.43261299169636747,
107
+ "nauc_recall_at_1000_diff1": 0.07915422635178945,
108
+ "nauc_precision_at_1_max": 0.07717968432014255,
109
+ "nauc_precision_at_1_std": -0.12128970836264957,
110
+ "nauc_precision_at_1_diff1": 0.2376849423041946,
111
+ "nauc_precision_at_3_max": 0.12725454226792474,
112
+ "nauc_precision_at_3_std": -0.08890301300521496,
113
+ "nauc_precision_at_3_diff1": 0.1641201145455273,
114
+ "nauc_precision_at_5_max": 0.14603065544378954,
115
+ "nauc_precision_at_5_std": -0.08393076636764327,
116
+ "nauc_precision_at_5_diff1": 0.13910498068961344,
117
+ "nauc_precision_at_10_max": 0.12810082741844767,
118
+ "nauc_precision_at_10_std": -0.048067030955217124,
119
+ "nauc_precision_at_10_diff1": 0.07837891235998415,
120
+ "nauc_precision_at_20_max": 0.11437627971859882,
121
+ "nauc_precision_at_20_std": -0.004305958722993792,
122
+ "nauc_precision_at_20_diff1": 0.0439594405302659,
123
+ "nauc_precision_at_100_max": 0.1001141171319629,
124
+ "nauc_precision_at_100_std": 0.07986023598989361,
125
+ "nauc_precision_at_100_diff1": 0.005135524982073292,
126
+ "nauc_precision_at_1000_max": 0.03918985896543267,
127
+ "nauc_precision_at_1000_std": 0.05646172908061015,
128
+ "nauc_precision_at_1000_diff1": -0.024630312703929947,
129
+ "nauc_mrr_at_1_max": 0.08027387784767404,
130
+ "nauc_mrr_at_1_std": -0.12085300208653037,
131
+ "nauc_mrr_at_1_diff1": 0.23561820952994392,
132
+ "nauc_mrr_at_3_max": 0.11287498313599616,
133
+ "nauc_mrr_at_3_std": -0.10884962293985108,
134
+ "nauc_mrr_at_3_diff1": 0.21427868061902244,
135
+ "nauc_mrr_at_5_max": 0.11827256695123471,
136
+ "nauc_mrr_at_5_std": -0.11263879264688029,
137
+ "nauc_mrr_at_5_diff1": 0.21565494364256044,
138
+ "nauc_mrr_at_10_max": 0.12083524938219001,
139
+ "nauc_mrr_at_10_std": -0.11108494883851726,
140
+ "nauc_mrr_at_10_diff1": 0.21431796043671433,
141
+ "nauc_mrr_at_20_max": 0.12149317651592814,
142
+ "nauc_mrr_at_20_std": -0.10862160916661551,
143
+ "nauc_mrr_at_20_diff1": 0.21345607390684102,
144
+ "nauc_mrr_at_100_max": 0.1205362471698749,
145
+ "nauc_mrr_at_100_std": -0.10862655767756421,
146
+ "nauc_mrr_at_100_diff1": 0.2141544593545465,
147
+ "nauc_mrr_at_1000_max": 0.12044666927428974,
148
+ "nauc_mrr_at_1000_std": -0.10866949129296426,
149
+ "nauc_mrr_at_1000_diff1": 0.2142416928431974,
150
+ "hit_rate_at_1": 0.26182,
151
+ "hit_rate_at_3": 0.50473,
152
+ "hit_rate_at_5": 0.6117,
153
+ "hit_rate_at_10": 0.72991,
154
+ "hit_rate_at_20": 0.81501,
155
+ "hit_rate_at_100": 0.9273,
156
+ "hit_rate_at_1000": 0.98522,
157
+ "main_score": 0.41913,
158
+ "hf_subset": "default",
159
+ "languages": [
160
+ "rus-Cyrl"
161
+ ]
162
+ }
163
+ ]
164
+ },
165
+ "evaluation_time": 14.55857229232788,
166
+ "kg_co2_emissions": null,
167
+ "date": null
168
+ }
169
+ }
results/mteb_rus_v1_1/raw_task_results/020_RUParaPhraserSTS.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RUParaPhraserSTS": {
3
+ "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4",
4
+ "task_name": "RUParaPhraserSTS",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "pearson": 0.5223370118317926,
10
+ "spearman": 0.5630408081111218,
11
+ "cosine_pearson": 0.5223370119935756,
12
+ "cosine_spearman": 0.56304081451451,
13
+ "manhattan_pearson": 0.5529272116081778,
14
+ "manhattan_spearman": 0.5649986102446305,
15
+ "euclidean_pearson": 0.5518461234982256,
16
+ "euclidean_spearman": 0.5630408064509842,
17
+ "main_score": 0.56304081451451,
18
+ "hf_subset": "default",
19
+ "languages": [
20
+ "rus-Cyrl"
21
+ ]
22
+ }
23
+ ]
24
+ },
25
+ "evaluation_time": 0.15336966514587402,
26
+ "kg_co2_emissions": null,
27
+ "date": null
28
+ }
29
+ }
results/mteb_rus_v1_1/raw_task_results/021_STS22.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "STS22": {
3
+ "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3",
4
+ "task_name": "STS22",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "pearson": 0.4648636834652642,
10
+ "spearman": 0.5723035046647749,
11
+ "cosine_pearson": 0.5211154750777177,
12
+ "cosine_spearman": 0.588935863512821,
13
+ "manhattan_pearson": 0.5196158473995662,
14
+ "manhattan_spearman": 0.5889510883169803,
15
+ "euclidean_pearson": 0.5195045294383644,
16
+ "euclidean_spearman": 0.5890089665323378,
17
+ "main_score": 0.588935863512821,
18
+ "hf_subset": "ru",
19
+ "languages": [
20
+ "rus-Cyrl"
21
+ ]
22
+ }
23
+ ]
24
+ },
25
+ "evaluation_time": 0.31448960304260254,
26
+ "kg_co2_emissions": null,
27
+ "date": null
28
+ }
29
+ }
results/mteb_rus_v1_1/raw_task_results/022_RuSTSBenchmarkSTS.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "RuSTSBenchmarkSTS": {
3
+ "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018",
4
+ "task_name": "RuSTSBenchmarkSTS",
5
+ "mteb_version": "2.12.30",
6
+ "scores": {
7
+ "test": [
8
+ {
9
+ "pearson": 0.7401651930584809,
10
+ "spearman": 0.7132891499629179,
11
+ "cosine_pearson": 0.7401651803110156,
12
+ "cosine_spearman": 0.7132969378142737,
13
+ "manhattan_pearson": 0.718634912366085,
14
+ "manhattan_spearman": 0.7103326354909923,
15
+ "euclidean_pearson": 0.7176931463770961,
16
+ "euclidean_spearman": 0.7132996435764726,
17
+ "main_score": 0.7132969378142737,
18
+ "hf_subset": "default",
19
+ "languages": [
20
+ "rus-Cyrl"
21
+ ]
22
+ }
23
+ ]
24
+ },
25
+ "evaluation_time": 0.09412121772766113,
26
+ "kg_co2_emissions": null,
27
+ "date": null
28
+ }
29
+ }
results/mteb_rus_v1_1/summary_compact.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": "MTEB(rus, v1.1)",
3
+ "metric": "mean_task_main_score",
4
+ "task_count": 23,
5
+ "completed_count": 23,
6
+ "failed_count": 0,
7
+ "mean_main_score": 0.5116301998770688,
8
+ "mean_main_score_pct": 51.16301998770688,
9
+ "task_type_scores": [
10
+ {
11
+ "task_type": "Classification",
12
+ "task_count": 9,
13
+ "mean_main_score": 0.5681099003283083,
14
+ "mean_main_score_pct": 56.81099003283083
15
+ },
16
+ {
17
+ "task_type": "Clustering",
18
+ "task_count": 3,
19
+ "mean_main_score": 0.5179856478158106,
20
+ "mean_main_score_pct": 51.79856478158106
21
+ },
22
+ {
23
+ "task_type": "MultilabelClassification",
24
+ "task_count": 2,
25
+ "mean_main_score": 0.35005150044002387,
26
+ "mean_main_score_pct": 35.005150044002384
27
+ },
28
+ {
29
+ "task_type": "PairClassification",
30
+ "task_count": 1,
31
+ "mean_main_score": 0.5250119340487216,
32
+ "mean_main_score_pct": 52.50119340487216
33
+ },
34
+ {
35
+ "task_type": "Reranking",
36
+ "task_count": 2,
37
+ "mean_main_score": 0.41878499999999996,
38
+ "mean_main_score_pct": 41.878499999999995
39
+ },
40
+ {
41
+ "task_type": "Retrieval",
42
+ "task_count": 3,
43
+ "mean_main_score": 0.39086333333333334,
44
+ "mean_main_score_pct": 39.086333333333336
45
+ },
46
+ {
47
+ "task_type": "STS",
48
+ "task_count": 3,
49
+ "mean_main_score": 0.6217578719472016,
50
+ "mean_main_score_pct": 62.17578719472016
51
+ }
52
+ ]
53
+ }
starse_static_meta.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "BorisTM/starse-512",
3
+ "model": "StaRSE-512",
4
+ "dim": 512,
5
+ "vocab_size": 120138,
6
+ "encoding": "packed_sign_bits_with_per_token_l2_norm",
7
+ "weights_layout": {
8
+ "packed_signs": [
9
+ 120138,
10
+ 64
11
+ ],
12
+ "norms": [
13
+ 120138
14
+ ]
15
+ },
16
+ "source_checkpoint": "starse-512 + 10000 joint qaware STE continuation (cultura_ru_only_45m, sw=0 bw=0.173 tkl=1.14 lr=3.77e-3 scale=20 mode=current cube=1.07e-4 up=7.89e-4)",
17
+ "packed_at_utc": "2026-05-23T09:27:12Z",
18
+ "rumteb_binary": 0.5116301998770688,
19
+ "rumteb_soft": 0.5056
20
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff