Hugging Face's logo

Spaces:
TIGER-Lab
/
MMEB-Leaderboard
Running on CPU Upgrade

App Files Community
202
MINGYISU commited on
Commit
8ef3917
·
1 Parent(s): 22869f7

v3 update final

Browse files
Files changed (6) hide show
  1. datasets.py +1 -1
  2. scores/LCO-Embedding-Omni-7B.json +0 -0
  3. scores/e5-omni-3B.json +0 -0
  4. scores/e5-omni-7B.json +0 -0
  5. scores/omni-embed-nemotron-3b.json +0 -0
  6. utils.py +1 -1
datasets.py CHANGED
@@ -36,7 +36,7 @@ DATASETS = {
36
  "V-MRET": ['QVHighlight', 'Charades-STA', 'MomentSeeker']
37
  },
38
  "audio": {
39
- "A-CLS": ["SpeechCommand", "UrbanSound8k", "ESC-50", "NSynth", "CREMA-D"],
40
  "A-RET": ['Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO']
41
  },
42
  "agent": {
 
36
  "V-MRET": ['QVHighlight', 'Charades-STA', 'MomentSeeker']
37
  },
38
  "audio": {
39
+ "A-CLS": ["SpeechCommands", "UrbanSound8K", "ESC-50", "NSynth", "CREMA-D"],
40
  "A-RET": ['Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO']
41
  },
42
  "agent": {
scores/LCO-Embedding-Omni-7B.json ADDED
The diff for this file is too large to render. See raw diff
 
scores/e5-omni-3B.json CHANGED
The diff for this file is too large to render. See raw diff
 
scores/e5-omni-7B.json CHANGED
The diff for this file is too large to render. See raw diff
 
scores/omni-embed-nemotron-3b.json ADDED
The diff for this file is too large to render. See raw diff
 
utils.py CHANGED
@@ -83,7 +83,7 @@ CITATION_BUTTON_TEXT = r"""@article{jiang2024vlm2vec,
83
  SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
84
  ## Please follow the guidelines in order to submit successfully. \n
85
  1. **Step 1️⃣:** Please refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n
86
- 2. **Step 2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v2.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v2.py))** to generate the final score sheet.
87
  - Adjust your model's configurations in the script before running it
88
  - Note the "model size" field is digits-only and is in Billions (B), so please convert it if yours is in different units/formats (e.x., "8" for 8 billion, "0.5" for 500 million).
89
  - If possible, please also add a contact method in case we want to reach you in the future
 
83
  SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
84
  ## Please follow the guidelines in order to submit successfully. \n
85
  1. **Step 1️⃣:** Please refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n
86
+ 2. **Step 2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v2.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v2.py))** to generate the final score sheet. (Use [report_score_v3.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v3.py) for v3 submission).
87
  - Adjust your model's configurations in the script before running it
88
  - Note the "model size" field is digits-only and is in Billions (B), so please convert it if yours is in different units/formats (e.x., "8" for 8 billion, "0.5" for 500 million).
89
  - If possible, please also add a contact method in case we want to reach you in the future