Johannes Janosovits commited on
Commit
96adbdd
·
1 Parent(s): 18160d5
Files changed (3) hide show
  1. app.py +115 -7
  2. coverage.py +248 -0
  3. coverage_summary.json +1217 -0
app.py CHANGED
@@ -1,11 +1,13 @@
1
  import json
2
  import os
3
  from datetime import datetime
4
- from pathlib import PurePosixPath
5
 
6
  from huggingface_hub import HfApi, hf_hub_download
7
  from shiny import App, reactive, render, ui
8
 
 
 
9
  DATASET_REPO = "MIMIR-AI-ROUTER/harness_evals"
10
  HF_BLOB_URL = f"https://huggingface.co/datasets/{DATASET_REPO}/blob/main"
11
  HF_TOKEN = os.getenv("HF_TOKEN")
@@ -51,6 +53,31 @@ tr.run-row.active td { background: var(--bg3); }
51
  .err { color: var(--accent); background: rgba(233,69,96,.1); padding: 10px 14px;
52
  border-radius: 4px; border-left: 3px solid var(--accent); margin-bottom: 16px; }
53
  .loading { color: var(--muted); padding: 40px; text-align: center; font-style: italic; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  """
55
 
56
  # ── data loading ──────────────────────────────────────────────────────────────
@@ -174,6 +201,14 @@ def load_runs(force: bool = False) -> tuple[list, str | None]:
174
 
175
  # ── UI ────────────────────────────────────────────────────────────────────────
176
 
 
 
 
 
 
 
 
 
177
  app_ui = ui.page_fluid(
178
  ui.tags.head(ui.tags.style(CSS)),
179
  ui.div(
@@ -192,13 +227,24 @@ app_ui = ui.page_fluid(
192
  {"class": "controls"},
193
  ui.input_action_button("refresh", "↺ Refresh", class_="btn"),
194
  ),
195
- ui.output_ui("runs_table"),
196
- ui.output_ui("task_detail"),
197
- ui.tags.script(
198
- "function selectRun(i){"
199
- " Shiny.setInputValue('sel_idx', i, {priority:'event'});"
200
- "}"
 
 
 
 
 
 
 
 
 
 
201
  ),
 
202
  ),
203
  )
204
 
@@ -321,5 +367,67 @@ def server(input, output, session):
321
  ),
322
  )
323
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
324
 
325
  app = App(app_ui, server)
 
1
  import json
2
  import os
3
  from datetime import datetime
4
+ from pathlib import Path, PurePosixPath
5
 
6
  from huggingface_hub import HfApi, hf_hub_download
7
  from shiny import App, reactive, render, ui
8
 
9
+ import coverage as cov
10
+
11
  DATASET_REPO = "MIMIR-AI-ROUTER/harness_evals"
12
  HF_BLOB_URL = f"https://huggingface.co/datasets/{DATASET_REPO}/blob/main"
13
  HF_TOKEN = os.getenv("HF_TOKEN")
 
53
  .err { color: var(--accent); background: rgba(233,69,96,.1); padding: 10px 14px;
54
  border-radius: 4px; border-left: 3px solid var(--accent); margin-bottom: 16px; }
55
  .loading { color: var(--muted); padding: 40px; text-align: center; font-style: italic; }
56
+ /* tabs */
57
+ .nav-tabs { border-bottom: 2px solid var(--border); margin-bottom: 20px; display: flex; gap: 4px; }
58
+ .nav-tab { background: none; border: none; color: var(--muted); padding: 8px 16px;
59
+ cursor: pointer; font-size: .9em; border-bottom: 2px solid transparent;
60
+ margin-bottom: -2px; }
61
+ .nav-tab:hover { color: var(--text); }
62
+ .nav-tab.active { color: var(--text); border-bottom-color: var(--accent); }
63
+ .tab-pane { display: none; }
64
+ .tab-pane.active { display: block; }
65
+ /* coverage */
66
+ .cov-stats { display: flex; gap: 12px; margin-bottom: 20px; flex-wrap: wrap; }
67
+ .stat-chip { background: var(--bg2); border-radius: 6px; padding: 10px 16px;
68
+ font-size: .85em; border-left: 3px solid var(--border); }
69
+ .stat-chip .n { font-size: 1.5em; font-weight: 700; display: block; }
70
+ .chip-ev { border-left-color: #4caf50; }
71
+ .chip-part { border-left-color: #ff9800; }
72
+ .chip-skip { border-left-color: #8888aa; }
73
+ .chip-none { border-left-color: var(--accent); }
74
+ .badge { display: inline-block; padding: 2px 7px; border-radius: 3px; font-size: .78em;
75
+ font-weight: 600; text-transform: uppercase; letter-spacing: .04em; }
76
+ .badge-ev { background: rgba(76,175,80,.18); color: #4caf50; }
77
+ .badge-part { background: rgba(255,152,0,.18); color: #ff9800; }
78
+ .badge-skip { background: rgba(136,136,170,.18); color: #8888aa; }
79
+ .badge-none { background: rgba(233,69,96,.12); color: var(--accent); }
80
+ .reason { color: var(--muted); font-size: .82em; font-style: italic; }
81
  """
82
 
83
  # ── data loading ──────────────────────────────────────────────────────────────
 
201
 
202
  # ── UI ────────────────────────────────────────────────────────────────────────
203
 
204
+ _TAB_JS = """
205
+ function switchTab(name) {
206
+ document.querySelectorAll('.nav-tab').forEach(t => t.classList.toggle('active', t.dataset.tab === name));
207
+ document.querySelectorAll('.tab-pane').forEach(p => p.classList.toggle('active', p.id === 'tab-' + name));
208
+ }
209
+ function selectRun(i) { Shiny.setInputValue('sel_idx', i, {priority:'event'}); }
210
+ """
211
+
212
  app_ui = ui.page_fluid(
213
  ui.tags.head(ui.tags.style(CSS)),
214
  ui.div(
 
227
  {"class": "controls"},
228
  ui.input_action_button("refresh", "↺ Refresh", class_="btn"),
229
  ),
230
+ # Tab bar
231
+ ui.div(
232
+ {"class": "nav-tabs"},
233
+ ui.tags.button("Results", {"class": "nav-tab active", "data-tab": "results", "onclick": "switchTab('results')"}),
234
+ ui.tags.button("Coverage", {"class": "nav-tab", "data-tab": "coverage", "onclick": "switchTab('coverage')"}),
235
+ ),
236
+ # Results tab
237
+ ui.div(
238
+ {"class": "tab-pane active", "id": "tab-results"},
239
+ ui.output_ui("runs_table"),
240
+ ui.output_ui("task_detail"),
241
+ ),
242
+ # Coverage tab
243
+ ui.div(
244
+ {"class": "tab-pane", "id": "tab-coverage"},
245
+ ui.output_ui("coverage_table"),
246
  ),
247
+ ui.tags.script(_TAB_JS),
248
  ),
249
  )
250
 
 
367
  ),
368
  )
369
 
370
+ @output
371
+ @render.ui
372
+ def coverage_table():
373
+ input.refresh() # refresh dependency
374
+ runs, _ = load_runs()
375
+
376
+ # Collect all evaluated task names from loaded runs
377
+ evaluated: set[str] = set()
378
+ for run in (runs or []):
379
+ for task in run["tasks"]:
380
+ evaluated.add(task["name"])
381
+
382
+ coverage = cov.compute_coverage(evaluated)
383
+ stats = cov.summary_stats(coverage)
384
+
385
+ _badge = {
386
+ "evaluated": ("ev", "Evaluated"),
387
+ "partial": ("part", "Partial"),
388
+ "skipped": ("skip", "Skipped"),
389
+ "not_started": ("none", "Not started"),
390
+ }
391
+
392
+ stat_chips = ui.div(
393
+ {"class": "cov-stats"},
394
+ ui.div({"class": "stat-chip chip-ev"}, ui.tags.span({"class": "n"}, str(stats["evaluated"])), "Evaluated"),
395
+ ui.div({"class": "stat-chip chip-part"}, ui.tags.span({"class": "n"}, str(stats["partial"])), "Partial"),
396
+ ui.div({"class": "stat-chip chip-none"}, ui.tags.span({"class": "n"}, str(stats["not_started"])), "Not started"),
397
+ ui.div({"class": "stat-chip chip-skip"}, ui.tags.span({"class": "n"}, str(stats["skipped"])), "Skipped"),
398
+ )
399
+
400
+ rows = []
401
+ for folder in cov.ALL_FOLDERS:
402
+ info = coverage[folder]
403
+ status = info["status"]
404
+ cls, label = _badge[status]
405
+ reason = info["skip_reason"] or ""
406
+ matched = info["matched_tasks"]
407
+
408
+ rows.append(ui.tags.tr(
409
+ ui.tags.td(folder),
410
+ ui.tags.td(ui.span({"class": f"badge badge-{cls}"}, label)),
411
+ ui.tags.td(
412
+ ui.span({"class": "reason"}, reason) if reason
413
+ else (ui.span({"class": "muted"}, ", ".join(matched[:5]) + ("…" if len(matched) > 5 else "")) if matched
414
+ else ui.span({"class": "muted"}, "—"))
415
+ ),
416
+ ))
417
+
418
+ return ui.div(
419
+ stat_chips,
420
+ ui.tags.table(
421
+ ui.tags.thead(
422
+ ui.tags.tr(
423
+ ui.tags.th("Task Folder"),
424
+ ui.tags.th("Status"),
425
+ ui.tags.th("Notes / Evaluated tasks"),
426
+ )
427
+ ),
428
+ ui.tags.tbody(*rows),
429
+ ),
430
+ )
431
+
432
 
433
  app = App(app_ui, server)
coverage.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Folder-level eval coverage for lm-evaluation-harness tasks.
3
+
4
+ SKIP_LIST: folders intentionally excluded from evaluation, with reasons.
5
+ Run as __main__ to regenerate coverage_summary.json from the HF dataset.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ # ── All task folders from lm_eval/tasks/ ─────────────────────────────────────
11
+ # (lm-evaluation-harness @ current HEAD, non-folder entries excluded)
12
+ ALL_FOLDERS: list[str] = [
13
+ "aclue", "acpbench", "aexams", "afrimgsm", "afrimmlu", "afrixnli",
14
+ "afrobench", "agieval", "aime", "alghafa", "anli", "arab_culture",
15
+ "arab_culture_completion", "arabic_leaderboard_complete",
16
+ "arabic_leaderboard_light", "arabicmmlu", "aradice", "arc", "arc_mt",
17
+ "arithmetic", "asdiv", "babi", "babilong", "bangla", "basque_bench",
18
+ "basqueglue", "bbh", "bbq", "bear", "belebele", "benchmarks", "bertaqa",
19
+ "bhs", "bigbench", "blimp", "blimp_nl", "c4", "cabbq", "careqa",
20
+ "catalan_bench", "ceval", "chartqa", "click", "cmmlu", "cnn_dailymail",
21
+ "code_x_glue", "commonsense_qa", "common_voice", "copal_id", "coqa",
22
+ "crows_pairs", "csatqa", "darija_bench", "darijahellaswag", "darijammlu",
23
+ "discrim_eval", "drop", "e2lmc", "egyhellaswag", "egymmlu", "eq_bench",
24
+ "esbbq", "eus_exams", "eus_proficiency", "eus_reading", "eus_trivia",
25
+ "evalita_llm", "fda", "fld", "french_bench", "galician_bench",
26
+ "glianorex", "global_mmlu", "global_piqa", "glue", "gpqa", "graphwalks",
27
+ "groundcocoa", "gsm8k", "gsm8k_platinum", "gsm_plus", "haerae",
28
+ "headqa", "hellaswag", "hendrycks_ethics", "hendrycks_math",
29
+ "histoires_morales", "hrm8k", "humaneval", "humaneval_infilling",
30
+ "icelandic_winogrande", "ifeval", "inverse_scaling",
31
+ "japanese_leaderboard", "jfinqa", "jsonschema_bench", "kbl", "kmmlu",
32
+ "kobest", "kormedmcqa", "lambada", "lambada_cloze",
33
+ "lambada_multilingual", "lambada_multilingual_stablelm", "leaderboard",
34
+ "libra", "lingoly", "llama3", "lm_syneval", "logiqa", "logiqa2",
35
+ "longbench", "longbench2", "mastermind", "mathqa", "mbpp", "mc_taco",
36
+ "med_concepts_qa", "meddialog", "mediqa_qa2019", "medmcqa",
37
+ "med_prescriptions", "medqa", "medtext", "med_text_classification",
38
+ "mela", "meqsum", "metabench", "mgsm", "mimic_repsum", "minerva_math",
39
+ "mlqa", "mmlu", "mmlu_pro", "mmlu-pro-plus", "mmlu_prox", "mmlu-redux",
40
+ "mmlu-redux-spanish", "mmlusr", "mmmu", "model_written_evals",
41
+ "moral_stories", "mts_dialog", "multiblimp", "mutual", "noreval",
42
+ "noticia", "nq_open", "okapi", "olaph", "openai-mmmlu", "openbookqa",
43
+ "paloma", "paws-x", "pile", "pile_10k", "piqa", "pisa", "polemo2",
44
+ "portuguese_bench", "prost", "pubmedqa", "qa4mre", "qasper", "race",
45
+ "realtoxicityprompts", "ruler", "sciq", "score", "scrolls",
46
+ "simple_cooccurrence_bias", "siqa", "slr_bench", "spanish_bench",
47
+ "squad_completion", "squadv2", "storycloze", "super_glue", "swag",
48
+ "swde", "tinyBenchmarks", "tmlu", "tmmluplus", "toxigen", "translation",
49
+ "triviaqa", "truthfulqa", "truthfulqa-multi", "turblimp", "turkishmmlu",
50
+ "ulqa", "unitxt", "unscramble", "webqs", "wikitext", "winogender",
51
+ "winogrande", "wmdp", "wmt2016", "wsc273", "xcopa", "xnli", "xnli_eu",
52
+ "xquad", "xstorycloze", "xwinograd", "zhoblimp",
53
+ ]
54
+
55
+ # ── Skip list ─────────────────────────────────────────────────────────────────
56
+ # Value = human-readable reason (fill in as needed).
57
+ SKIP_LIST: dict[str, str] = {
58
+ # ── Non-English / regional ────────────────────────────────────────────────
59
+ "aclue": "",
60
+ "afrimgsm": "",
61
+ "afrimmlu": "",
62
+ "afrixnli": "",
63
+ "afrobench": "",
64
+ "alghafa": "",
65
+ "arab_culture": "",
66
+ "arab_culture_completion": "",
67
+ "arabic_leaderboard_complete": "",
68
+ "arabic_leaderboard_light": "",
69
+ "arabicmmlu": "",
70
+ "aradice": "",
71
+ "arc_mt": "",
72
+ "bangla": "",
73
+ "basque_bench": "",
74
+ "basqueglue": "",
75
+ "belebele": "",
76
+ "bertaqa": "",
77
+ "bhs": "",
78
+ "cabbq": "",
79
+ "catalan_bench": "",
80
+ "ceval": "",
81
+ "cmmlu": "",
82
+ "copal_id": "",
83
+ "darija_bench": "",
84
+ "darijahellaswag": "",
85
+ "darijammlu": "",
86
+ "egyhellaswag": "",
87
+ "egymmlu": "",
88
+ "esbbq": "",
89
+ "eus_exams": "",
90
+ "eus_proficiency": "",
91
+ "eus_reading": "",
92
+ "eus_trivia": "",
93
+ "evalita_llm": "",
94
+ "french_bench": "",
95
+ "galician_bench": "",
96
+ "haerae": "",
97
+ "histoires_morales": "",
98
+ "hrm8k": "",
99
+ "icelandic_winogrande": "",
100
+ "japanese_leaderboard": "",
101
+ "jfinqa": "",
102
+ "kbl": "",
103
+ "kmmlu": "",
104
+ "kobest": "",
105
+ "kormedmcqa": "",
106
+ "lambada_multilingual": "",
107
+ "lambada_multilingual_stablelm": "",
108
+ "libra": "",
109
+ "lingoly": "",
110
+ "mela": "",
111
+ "mgsm": "",
112
+ "mlqa": "",
113
+ "noreval": "",
114
+ "noticia": "",
115
+ "okapi": "",
116
+ "openai-mmmlu": "",
117
+ "paws-x": "",
118
+ "polemo2": "",
119
+ "portuguese_bench": "",
120
+ "spanish_bench": "",
121
+ "tmlu": "",
122
+ "tmmluplus": "",
123
+ "turkishmmlu": "",
124
+ "turblimp": "",
125
+ "xcopa": "",
126
+ "xnli_eu": "",
127
+ "xwinograd": "",
128
+ "zhoblimp": "",
129
+ # ── Code execution required ───────────────────────────────────────────────
130
+ "code_x_glue": "",
131
+ "humaneval": "",
132
+ "humaneval_infilling": "",
133
+ "mbpp": "",
134
+ # ── Aggregate / meta benchmark collections ────────────────────────────────
135
+ "benchmarks": "",
136
+ "leaderboard": "",
137
+ "llama3": "",
138
+ "metabench": "",
139
+ "tinyBenchmarks": "",
140
+ # ── Large corpus / perplexity-only ────────────────────────────────────────
141
+ "c4": "",
142
+ "paloma": "",
143
+ "pile": "",
144
+ "pile_10k": "",
145
+ "wikitext": "",
146
+ }
147
+
148
+ # ── Core logic ────────────────────────────────────────────────────────────────
149
+
150
+ def _folder_for_task(task: str, folder_set: set[str]) -> str | None:
151
+ """Return the longest folder name that is a prefix of task (with separator)."""
152
+ best: str | None = None
153
+ for folder in folder_set:
154
+ if task == folder or task.startswith(folder + "_") or task.startswith(folder + "-"):
155
+ if best is None or len(folder) > len(best):
156
+ best = folder
157
+ return best
158
+
159
+
160
+ def compute_coverage(evaluated_tasks: set[str]) -> dict[str, dict]:
161
+ """
162
+ Returns a dict mapping each folder name to its coverage info:
163
+ status: "evaluated" | "partial" | "skipped" | "not_started"
164
+ skip_reason: str | None
165
+ matched_tasks: list[str] (evaluated task names mapped to this folder)
166
+ """
167
+ folder_set = set(ALL_FOLDERS)
168
+
169
+ # Map every evaluated task to its folder
170
+ folder_hits: dict[str, list[str]] = {f: [] for f in ALL_FOLDERS}
171
+ for task in evaluated_tasks:
172
+ folder = _folder_for_task(task, folder_set)
173
+ if folder:
174
+ folder_hits[folder].append(task)
175
+
176
+ result: dict[str, dict] = {}
177
+ for folder in ALL_FOLDERS:
178
+ hits = folder_hits[folder]
179
+ if folder in SKIP_LIST:
180
+ status = "skipped"
181
+ elif not hits:
182
+ status = "not_started"
183
+ elif folder in evaluated_tasks:
184
+ # The group task itself was evaluated → full run
185
+ status = "evaluated"
186
+ else:
187
+ status = "partial"
188
+
189
+ result[folder] = {
190
+ "status": status,
191
+ "skip_reason": SKIP_LIST.get(folder),
192
+ "matched_tasks": sorted(hits),
193
+ }
194
+
195
+ return result
196
+
197
+
198
+ def summary_stats(coverage: dict[str, dict]) -> dict[str, int]:
199
+ counts: dict[str, int] = {"evaluated": 0, "partial": 0, "skipped": 0, "not_started": 0}
200
+ for info in coverage.values():
201
+ counts[info["status"]] += 1
202
+ return counts
203
+
204
+
205
+ # ── CLI: regenerate coverage_summary.json ────────────────────────────────────
206
+
207
+ if __name__ == "__main__":
208
+ import json, os
209
+ from datetime import datetime, timezone
210
+ from pathlib import Path
211
+ from pathlib import PurePosixPath
212
+ from huggingface_hub import HfApi, hf_hub_download
213
+
214
+ REPO = "MIMIR-AI-ROUTER/harness_evals"
215
+ TOKEN = os.getenv("HF_TOKEN")
216
+
217
+ print("Fetching file list...")
218
+ api = HfApi(token=TOKEN)
219
+ all_files = sorted(api.list_repo_files(REPO, repo_type="dataset"))
220
+ result_files = [
221
+ f for f in all_files
222
+ if PurePosixPath(f).name.startswith("results_") and f.endswith(".json")
223
+ ]
224
+ print(f" {len(result_files)} result files found")
225
+
226
+ evaluated: set[str] = set()
227
+ for filename in result_files:
228
+ local = hf_hub_download(REPO, filename, repo_type="dataset", token=TOKEN)
229
+ data = json.load(open(local))
230
+ evaluated.update(data.get("results", {}).keys())
231
+ print(f" {len(evaluated)} unique evaluated tasks")
232
+
233
+ coverage = compute_coverage(evaluated)
234
+ stats = summary_stats(coverage)
235
+
236
+ out = {
237
+ "generated_at": datetime.now(timezone.utc).isoformat(),
238
+ "stats": stats,
239
+ "folders": coverage,
240
+ }
241
+
242
+ dest = Path(__file__).parent / "coverage_summary.json"
243
+ dest.write_text(json.dumps(out, indent=2))
244
+ print(f"\nWrote {dest}")
245
+ print(f" evaluated: {stats['evaluated']}")
246
+ print(f" partial: {stats['partial']}")
247
+ print(f" skipped: {stats['skipped']}")
248
+ print(f" not_started: {stats['not_started']}")
coverage_summary.json ADDED
@@ -0,0 +1,1217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "generated_at": "2026-04-21T23:29:28.778844+00:00",
3
+ "stats": {
4
+ "evaluated": 17,
5
+ "partial": 16,
6
+ "skipped": 84,
7
+ "not_started": 90
8
+ },
9
+ "folders": {
10
+ "aclue": {
11
+ "status": "skipped",
12
+ "skip_reason": "",
13
+ "matched_tasks": []
14
+ },
15
+ "acpbench": {
16
+ "status": "not_started",
17
+ "skip_reason": null,
18
+ "matched_tasks": []
19
+ },
20
+ "aexams": {
21
+ "status": "not_started",
22
+ "skip_reason": null,
23
+ "matched_tasks": []
24
+ },
25
+ "afrimgsm": {
26
+ "status": "skipped",
27
+ "skip_reason": "",
28
+ "matched_tasks": []
29
+ },
30
+ "afrimmlu": {
31
+ "status": "skipped",
32
+ "skip_reason": "",
33
+ "matched_tasks": []
34
+ },
35
+ "afrixnli": {
36
+ "status": "skipped",
37
+ "skip_reason": "",
38
+ "matched_tasks": []
39
+ },
40
+ "afrobench": {
41
+ "status": "skipped",
42
+ "skip_reason": "",
43
+ "matched_tasks": []
44
+ },
45
+ "agieval": {
46
+ "status": "not_started",
47
+ "skip_reason": null,
48
+ "matched_tasks": []
49
+ },
50
+ "aime": {
51
+ "status": "not_started",
52
+ "skip_reason": null,
53
+ "matched_tasks": []
54
+ },
55
+ "alghafa": {
56
+ "status": "skipped",
57
+ "skip_reason": "",
58
+ "matched_tasks": []
59
+ },
60
+ "anli": {
61
+ "status": "not_started",
62
+ "skip_reason": null,
63
+ "matched_tasks": []
64
+ },
65
+ "arab_culture": {
66
+ "status": "skipped",
67
+ "skip_reason": "",
68
+ "matched_tasks": []
69
+ },
70
+ "arab_culture_completion": {
71
+ "status": "skipped",
72
+ "skip_reason": "",
73
+ "matched_tasks": []
74
+ },
75
+ "arabic_leaderboard_complete": {
76
+ "status": "skipped",
77
+ "skip_reason": "",
78
+ "matched_tasks": []
79
+ },
80
+ "arabic_leaderboard_light": {
81
+ "status": "skipped",
82
+ "skip_reason": "",
83
+ "matched_tasks": []
84
+ },
85
+ "arabicmmlu": {
86
+ "status": "skipped",
87
+ "skip_reason": "",
88
+ "matched_tasks": []
89
+ },
90
+ "aradice": {
91
+ "status": "skipped",
92
+ "skip_reason": "",
93
+ "matched_tasks": []
94
+ },
95
+ "arc": {
96
+ "status": "not_started",
97
+ "skip_reason": null,
98
+ "matched_tasks": []
99
+ },
100
+ "arc_mt": {
101
+ "status": "skipped",
102
+ "skip_reason": "",
103
+ "matched_tasks": []
104
+ },
105
+ "arithmetic": {
106
+ "status": "not_started",
107
+ "skip_reason": null,
108
+ "matched_tasks": []
109
+ },
110
+ "asdiv": {
111
+ "status": "evaluated",
112
+ "skip_reason": null,
113
+ "matched_tasks": [
114
+ "asdiv"
115
+ ]
116
+ },
117
+ "babi": {
118
+ "status": "evaluated",
119
+ "skip_reason": null,
120
+ "matched_tasks": [
121
+ "babi"
122
+ ]
123
+ },
124
+ "babilong": {
125
+ "status": "not_started",
126
+ "skip_reason": null,
127
+ "matched_tasks": []
128
+ },
129
+ "bangla": {
130
+ "status": "skipped",
131
+ "skip_reason": "",
132
+ "matched_tasks": []
133
+ },
134
+ "basque_bench": {
135
+ "status": "skipped",
136
+ "skip_reason": "",
137
+ "matched_tasks": []
138
+ },
139
+ "basqueglue": {
140
+ "status": "skipped",
141
+ "skip_reason": "",
142
+ "matched_tasks": []
143
+ },
144
+ "bbh": {
145
+ "status": "not_started",
146
+ "skip_reason": null,
147
+ "matched_tasks": []
148
+ },
149
+ "bbq": {
150
+ "status": "not_started",
151
+ "skip_reason": null,
152
+ "matched_tasks": []
153
+ },
154
+ "bear": {
155
+ "status": "partial",
156
+ "skip_reason": null,
157
+ "matched_tasks": [
158
+ "bear_big"
159
+ ]
160
+ },
161
+ "belebele": {
162
+ "status": "skipped",
163
+ "skip_reason": "",
164
+ "matched_tasks": []
165
+ },
166
+ "benchmarks": {
167
+ "status": "skipped",
168
+ "skip_reason": "",
169
+ "matched_tasks": []
170
+ },
171
+ "bertaqa": {
172
+ "status": "skipped",
173
+ "skip_reason": "",
174
+ "matched_tasks": []
175
+ },
176
+ "bhs": {
177
+ "status": "skipped",
178
+ "skip_reason": "",
179
+ "matched_tasks": []
180
+ },
181
+ "bigbench": {
182
+ "status": "partial",
183
+ "skip_reason": null,
184
+ "matched_tasks": [
185
+ "bigbench_strategyqa_multiple_choice"
186
+ ]
187
+ },
188
+ "blimp": {
189
+ "status": "evaluated",
190
+ "skip_reason": null,
191
+ "matched_tasks": [
192
+ "blimp",
193
+ "blimp_adjunct_island",
194
+ "blimp_anaphor_gender_agreement",
195
+ "blimp_anaphor_number_agreement",
196
+ "blimp_animate_subject_passive",
197
+ "blimp_animate_subject_trans",
198
+ "blimp_causative",
199
+ "blimp_complex_NP_island",
200
+ "blimp_coordinate_structure_constraint_complex_left_branch",
201
+ "blimp_coordinate_structure_constraint_object_extraction",
202
+ "blimp_determiner_noun_agreement_1",
203
+ "blimp_determiner_noun_agreement_2",
204
+ "blimp_determiner_noun_agreement_irregular_1",
205
+ "blimp_determiner_noun_agreement_irregular_2",
206
+ "blimp_determiner_noun_agreement_with_adj_2",
207
+ "blimp_determiner_noun_agreement_with_adj_irregular_1",
208
+ "blimp_determiner_noun_agreement_with_adj_irregular_2",
209
+ "blimp_determiner_noun_agreement_with_adjective_1",
210
+ "blimp_distractor_agreement_relational_noun",
211
+ "blimp_distractor_agreement_relative_clause",
212
+ "blimp_drop_argument",
213
+ "blimp_ellipsis_n_bar_1",
214
+ "blimp_ellipsis_n_bar_2",
215
+ "blimp_existential_there_object_raising",
216
+ "blimp_existential_there_quantifiers_1",
217
+ "blimp_existential_there_quantifiers_2",
218
+ "blimp_existential_there_subject_raising",
219
+ "blimp_expletive_it_object_raising",
220
+ "blimp_inchoative",
221
+ "blimp_intransitive",
222
+ "blimp_irregular_past_participle_adjectives",
223
+ "blimp_irregular_past_participle_verbs",
224
+ "blimp_irregular_plural_subject_verb_agreement_1",
225
+ "blimp_irregular_plural_subject_verb_agreement_2",
226
+ "blimp_left_branch_island_echo_question",
227
+ "blimp_left_branch_island_simple_question",
228
+ "blimp_matrix_question_npi_licensor_present",
229
+ "blimp_npi_present_1",
230
+ "blimp_npi_present_2",
231
+ "blimp_only_npi_licensor_present",
232
+ "blimp_only_npi_scope",
233
+ "blimp_passive_1",
234
+ "blimp_passive_2",
235
+ "blimp_principle_A_c_command",
236
+ "blimp_principle_A_case_1",
237
+ "blimp_principle_A_case_2",
238
+ "blimp_principle_A_domain_1",
239
+ "blimp_principle_A_domain_2",
240
+ "blimp_principle_A_domain_3",
241
+ "blimp_principle_A_reconstruction",
242
+ "blimp_regular_plural_subject_verb_agreement_1",
243
+ "blimp_regular_plural_subject_verb_agreement_2",
244
+ "blimp_sentential_negation_npi_licensor_present",
245
+ "blimp_sentential_negation_npi_scope",
246
+ "blimp_sentential_subject_island",
247
+ "blimp_superlative_quantifiers_1",
248
+ "blimp_superlative_quantifiers_2",
249
+ "blimp_tough_vs_raising_1",
250
+ "blimp_tough_vs_raising_2",
251
+ "blimp_transitive",
252
+ "blimp_wh_island",
253
+ "blimp_wh_questions_object_gap",
254
+ "blimp_wh_questions_subject_gap",
255
+ "blimp_wh_questions_subject_gap_long_distance",
256
+ "blimp_wh_vs_that_no_gap",
257
+ "blimp_wh_vs_that_no_gap_long_distance",
258
+ "blimp_wh_vs_that_with_gap",
259
+ "blimp_wh_vs_that_with_gap_long_distance"
260
+ ]
261
+ },
262
+ "blimp_nl": {
263
+ "status": "not_started",
264
+ "skip_reason": null,
265
+ "matched_tasks": []
266
+ },
267
+ "c4": {
268
+ "status": "skipped",
269
+ "skip_reason": "",
270
+ "matched_tasks": []
271
+ },
272
+ "cabbq": {
273
+ "status": "skipped",
274
+ "skip_reason": "",
275
+ "matched_tasks": []
276
+ },
277
+ "careqa": {
278
+ "status": "not_started",
279
+ "skip_reason": null,
280
+ "matched_tasks": []
281
+ },
282
+ "catalan_bench": {
283
+ "status": "skipped",
284
+ "skip_reason": "",
285
+ "matched_tasks": []
286
+ },
287
+ "ceval": {
288
+ "status": "skipped",
289
+ "skip_reason": "",
290
+ "matched_tasks": []
291
+ },
292
+ "chartqa": {
293
+ "status": "not_started",
294
+ "skip_reason": null,
295
+ "matched_tasks": []
296
+ },
297
+ "click": {
298
+ "status": "not_started",
299
+ "skip_reason": null,
300
+ "matched_tasks": []
301
+ },
302
+ "cmmlu": {
303
+ "status": "skipped",
304
+ "skip_reason": "",
305
+ "matched_tasks": []
306
+ },
307
+ "cnn_dailymail": {
308
+ "status": "not_started",
309
+ "skip_reason": null,
310
+ "matched_tasks": []
311
+ },
312
+ "code_x_glue": {
313
+ "status": "skipped",
314
+ "skip_reason": "",
315
+ "matched_tasks": []
316
+ },
317
+ "commonsense_qa": {
318
+ "status": "not_started",
319
+ "skip_reason": null,
320
+ "matched_tasks": []
321
+ },
322
+ "common_voice": {
323
+ "status": "not_started",
324
+ "skip_reason": null,
325
+ "matched_tasks": []
326
+ },
327
+ "copal_id": {
328
+ "status": "skipped",
329
+ "skip_reason": "",
330
+ "matched_tasks": []
331
+ },
332
+ "coqa": {
333
+ "status": "not_started",
334
+ "skip_reason": null,
335
+ "matched_tasks": []
336
+ },
337
+ "crows_pairs": {
338
+ "status": "partial",
339
+ "skip_reason": null,
340
+ "matched_tasks": [
341
+ "crows_pairs_english"
342
+ ]
343
+ },
344
+ "csatqa": {
345
+ "status": "not_started",
346
+ "skip_reason": null,
347
+ "matched_tasks": []
348
+ },
349
+ "darija_bench": {
350
+ "status": "skipped",
351
+ "skip_reason": "",
352
+ "matched_tasks": []
353
+ },
354
+ "darijahellaswag": {
355
+ "status": "skipped",
356
+ "skip_reason": "",
357
+ "matched_tasks": []
358
+ },
359
+ "darijammlu": {
360
+ "status": "skipped",
361
+ "skip_reason": "",
362
+ "matched_tasks": []
363
+ },
364
+ "discrim_eval": {
365
+ "status": "partial",
366
+ "skip_reason": null,
367
+ "matched_tasks": [
368
+ "discrim_eval_explicit",
369
+ "discrim_eval_implicit"
370
+ ]
371
+ },
372
+ "drop": {
373
+ "status": "not_started",
374
+ "skip_reason": null,
375
+ "matched_tasks": []
376
+ },
377
+ "e2lmc": {
378
+ "status": "not_started",
379
+ "skip_reason": null,
380
+ "matched_tasks": []
381
+ },
382
+ "egyhellaswag": {
383
+ "status": "skipped",
384
+ "skip_reason": "",
385
+ "matched_tasks": []
386
+ },
387
+ "egymmlu": {
388
+ "status": "skipped",
389
+ "skip_reason": "",
390
+ "matched_tasks": []
391
+ },
392
+ "eq_bench": {
393
+ "status": "not_started",
394
+ "skip_reason": null,
395
+ "matched_tasks": []
396
+ },
397
+ "esbbq": {
398
+ "status": "skipped",
399
+ "skip_reason": "",
400
+ "matched_tasks": []
401
+ },
402
+ "eus_exams": {
403
+ "status": "skipped",
404
+ "skip_reason": "",
405
+ "matched_tasks": []
406
+ },
407
+ "eus_proficiency": {
408
+ "status": "skipped",
409
+ "skip_reason": "",
410
+ "matched_tasks": []
411
+ },
412
+ "eus_reading": {
413
+ "status": "skipped",
414
+ "skip_reason": "",
415
+ "matched_tasks": []
416
+ },
417
+ "eus_trivia": {
418
+ "status": "skipped",
419
+ "skip_reason": "",
420
+ "matched_tasks": []
421
+ },
422
+ "evalita_llm": {
423
+ "status": "skipped",
424
+ "skip_reason": "",
425
+ "matched_tasks": []
426
+ },
427
+ "fda": {
428
+ "status": "evaluated",
429
+ "skip_reason": null,
430
+ "matched_tasks": [
431
+ "fda"
432
+ ]
433
+ },
434
+ "fld": {
435
+ "status": "partial",
436
+ "skip_reason": null,
437
+ "matched_tasks": [
438
+ "fld_default"
439
+ ]
440
+ },
441
+ "french_bench": {
442
+ "status": "skipped",
443
+ "skip_reason": "",
444
+ "matched_tasks": []
445
+ },
446
+ "galician_bench": {
447
+ "status": "skipped",
448
+ "skip_reason": "",
449
+ "matched_tasks": []
450
+ },
451
+ "glianorex": {
452
+ "status": "partial",
453
+ "skip_reason": null,
454
+ "matched_tasks": [
455
+ "glianorex_en"
456
+ ]
457
+ },
458
+ "global_mmlu": {
459
+ "status": "not_started",
460
+ "skip_reason": null,
461
+ "matched_tasks": []
462
+ },
463
+ "global_piqa": {
464
+ "status": "not_started",
465
+ "skip_reason": null,
466
+ "matched_tasks": []
467
+ },
468
+ "glue": {
469
+ "status": "not_started",
470
+ "skip_reason": null,
471
+ "matched_tasks": []
472
+ },
473
+ "gpqa": {
474
+ "status": "not_started",
475
+ "skip_reason": null,
476
+ "matched_tasks": []
477
+ },
478
+ "graphwalks": {
479
+ "status": "not_started",
480
+ "skip_reason": null,
481
+ "matched_tasks": []
482
+ },
483
+ "groundcocoa": {
484
+ "status": "evaluated",
485
+ "skip_reason": null,
486
+ "matched_tasks": [
487
+ "groundcocoa"
488
+ ]
489
+ },
490
+ "gsm8k": {
491
+ "status": "not_started",
492
+ "skip_reason": null,
493
+ "matched_tasks": []
494
+ },
495
+ "gsm8k_platinum": {
496
+ "status": "not_started",
497
+ "skip_reason": null,
498
+ "matched_tasks": []
499
+ },
500
+ "gsm_plus": {
501
+ "status": "evaluated",
502
+ "skip_reason": null,
503
+ "matched_tasks": [
504
+ "gsm_plus",
505
+ "gsm_plus_mini"
506
+ ]
507
+ },
508
+ "haerae": {
509
+ "status": "skipped",
510
+ "skip_reason": "",
511
+ "matched_tasks": []
512
+ },
513
+ "headqa": {
514
+ "status": "partial",
515
+ "skip_reason": null,
516
+ "matched_tasks": [
517
+ "headqa_en"
518
+ ]
519
+ },
520
+ "hellaswag": {
521
+ "status": "not_started",
522
+ "skip_reason": null,
523
+ "matched_tasks": []
524
+ },
525
+ "hendrycks_ethics": {
526
+ "status": "not_started",
527
+ "skip_reason": null,
528
+ "matched_tasks": []
529
+ },
530
+ "hendrycks_math": {
531
+ "status": "not_started",
532
+ "skip_reason": null,
533
+ "matched_tasks": []
534
+ },
535
+ "histoires_morales": {
536
+ "status": "skipped",
537
+ "skip_reason": "",
538
+ "matched_tasks": []
539
+ },
540
+ "hrm8k": {
541
+ "status": "skipped",
542
+ "skip_reason": "",
543
+ "matched_tasks": []
544
+ },
545
+ "humaneval": {
546
+ "status": "skipped",
547
+ "skip_reason": "",
548
+ "matched_tasks": []
549
+ },
550
+ "humaneval_infilling": {
551
+ "status": "skipped",
552
+ "skip_reason": "",
553
+ "matched_tasks": []
554
+ },
555
+ "icelandic_winogrande": {
556
+ "status": "skipped",
557
+ "skip_reason": "",
558
+ "matched_tasks": []
559
+ },
560
+ "ifeval": {
561
+ "status": "not_started",
562
+ "skip_reason": null,
563
+ "matched_tasks": []
564
+ },
565
+ "inverse_scaling": {
566
+ "status": "not_started",
567
+ "skip_reason": null,
568
+ "matched_tasks": []
569
+ },
570
+ "japanese_leaderboard": {
571
+ "status": "skipped",
572
+ "skip_reason": "",
573
+ "matched_tasks": []
574
+ },
575
+ "jfinqa": {
576
+ "status": "skipped",
577
+ "skip_reason": "",
578
+ "matched_tasks": []
579
+ },
580
+ "jsonschema_bench": {
581
+ "status": "partial",
582
+ "skip_reason": null,
583
+ "matched_tasks": [
584
+ "jsonschema_bench_easy",
585
+ "jsonschema_bench_hard",
586
+ "jsonschema_bench_medium"
587
+ ]
588
+ },
589
+ "kbl": {
590
+ "status": "skipped",
591
+ "skip_reason": "",
592
+ "matched_tasks": []
593
+ },
594
+ "kmmlu": {
595
+ "status": "skipped",
596
+ "skip_reason": "",
597
+ "matched_tasks": []
598
+ },
599
+ "kobest": {
600
+ "status": "skipped",
601
+ "skip_reason": "",
602
+ "matched_tasks": []
603
+ },
604
+ "kormedmcqa": {
605
+ "status": "skipped",
606
+ "skip_reason": "",
607
+ "matched_tasks": []
608
+ },
609
+ "lambada": {
610
+ "status": "partial",
611
+ "skip_reason": null,
612
+ "matched_tasks": [
613
+ "lambada_standard"
614
+ ]
615
+ },
616
+ "lambada_cloze": {
617
+ "status": "not_started",
618
+ "skip_reason": null,
619
+ "matched_tasks": []
620
+ },
621
+ "lambada_multilingual": {
622
+ "status": "skipped",
623
+ "skip_reason": "",
624
+ "matched_tasks": []
625
+ },
626
+ "lambada_multilingual_stablelm": {
627
+ "status": "skipped",
628
+ "skip_reason": "",
629
+ "matched_tasks": []
630
+ },
631
+ "leaderboard": {
632
+ "status": "skipped",
633
+ "skip_reason": "",
634
+ "matched_tasks": []
635
+ },
636
+ "libra": {
637
+ "status": "skipped",
638
+ "skip_reason": "",
639
+ "matched_tasks": []
640
+ },
641
+ "lingoly": {
642
+ "status": "skipped",
643
+ "skip_reason": "",
644
+ "matched_tasks": []
645
+ },
646
+ "llama3": {
647
+ "status": "skipped",
648
+ "skip_reason": "",
649
+ "matched_tasks": []
650
+ },
651
+ "lm_syneval": {
652
+ "status": "not_started",
653
+ "skip_reason": null,
654
+ "matched_tasks": []
655
+ },
656
+ "logiqa": {
657
+ "status": "not_started",
658
+ "skip_reason": null,
659
+ "matched_tasks": []
660
+ },
661
+ "logiqa2": {
662
+ "status": "not_started",
663
+ "skip_reason": null,
664
+ "matched_tasks": []
665
+ },
666
+ "longbench": {
667
+ "status": "not_started",
668
+ "skip_reason": null,
669
+ "matched_tasks": []
670
+ },
671
+ "longbench2": {
672
+ "status": "not_started",
673
+ "skip_reason": null,
674
+ "matched_tasks": []
675
+ },
676
+ "mastermind": {
677
+ "status": "partial",
678
+ "skip_reason": null,
679
+ "matched_tasks": [
680
+ "mastermind_24_easy",
681
+ "mastermind_24_hard",
682
+ "mastermind_35_easy",
683
+ "mastermind_35_hard",
684
+ "mastermind_46_easy",
685
+ "mastermind_46_hard"
686
+ ]
687
+ },
688
+ "mathqa": {
689
+ "status": "not_started",
690
+ "skip_reason": null,
691
+ "matched_tasks": []
692
+ },
693
+ "mbpp": {
694
+ "status": "skipped",
695
+ "skip_reason": "",
696
+ "matched_tasks": []
697
+ },
698
+ "mc_taco": {
699
+ "status": "not_started",
700
+ "skip_reason": null,
701
+ "matched_tasks": []
702
+ },
703
+ "med_concepts_qa": {
704
+ "status": "evaluated",
705
+ "skip_reason": null,
706
+ "matched_tasks": [
707
+ "med_concepts_qa",
708
+ "med_concepts_qa_atc",
709
+ "med_concepts_qa_atc_easy",
710
+ "med_concepts_qa_atc_hard",
711
+ "med_concepts_qa_atc_medium",
712
+ "med_concepts_qa_icd10cm",
713
+ "med_concepts_qa_icd10cm_easy",
714
+ "med_concepts_qa_icd10cm_hard",
715
+ "med_concepts_qa_icd10cm_medium",
716
+ "med_concepts_qa_icd10proc",
717
+ "med_concepts_qa_icd10proc_easy",
718
+ "med_concepts_qa_icd10proc_hard",
719
+ "med_concepts_qa_icd10proc_medium",
720
+ "med_concepts_qa_icd9cm",
721
+ "med_concepts_qa_icd9cm_easy",
722
+ "med_concepts_qa_icd9cm_hard",
723
+ "med_concepts_qa_icd9cm_medium",
724
+ "med_concepts_qa_icd9proc",
725
+ "med_concepts_qa_icd9proc_easy",
726
+ "med_concepts_qa_icd9proc_hard",
727
+ "med_concepts_qa_icd9proc_medium"
728
+ ]
729
+ },
730
+ "meddialog": {
731
+ "status": "not_started",
732
+ "skip_reason": null,
733
+ "matched_tasks": []
734
+ },
735
+ "mediqa_qa2019": {
736
+ "status": "not_started",
737
+ "skip_reason": null,
738
+ "matched_tasks": []
739
+ },
740
+ "medmcqa": {
741
+ "status": "not_started",
742
+ "skip_reason": null,
743
+ "matched_tasks": []
744
+ },
745
+ "med_prescriptions": {
746
+ "status": "not_started",
747
+ "skip_reason": null,
748
+ "matched_tasks": []
749
+ },
750
+ "medqa": {
751
+ "status": "not_started",
752
+ "skip_reason": null,
753
+ "matched_tasks": []
754
+ },
755
+ "medtext": {
756
+ "status": "not_started",
757
+ "skip_reason": null,
758
+ "matched_tasks": []
759
+ },
760
+ "med_text_classification": {
761
+ "status": "not_started",
762
+ "skip_reason": null,
763
+ "matched_tasks": []
764
+ },
765
+ "mela": {
766
+ "status": "skipped",
767
+ "skip_reason": "",
768
+ "matched_tasks": []
769
+ },
770
+ "meqsum": {
771
+ "status": "not_started",
772
+ "skip_reason": null,
773
+ "matched_tasks": []
774
+ },
775
+ "metabench": {
776
+ "status": "skipped",
777
+ "skip_reason": "",
778
+ "matched_tasks": []
779
+ },
780
+ "mgsm": {
781
+ "status": "skipped",
782
+ "skip_reason": "",
783
+ "matched_tasks": []
784
+ },
785
+ "mimic_repsum": {
786
+ "status": "not_started",
787
+ "skip_reason": null,
788
+ "matched_tasks": []
789
+ },
790
+ "minerva_math": {
791
+ "status": "not_started",
792
+ "skip_reason": null,
793
+ "matched_tasks": []
794
+ },
795
+ "mlqa": {
796
+ "status": "skipped",
797
+ "skip_reason": "",
798
+ "matched_tasks": []
799
+ },
800
+ "mmlu": {
801
+ "status": "not_started",
802
+ "skip_reason": null,
803
+ "matched_tasks": []
804
+ },
805
+ "mmlu_pro": {
806
+ "status": "not_started",
807
+ "skip_reason": null,
808
+ "matched_tasks": []
809
+ },
810
+ "mmlu-pro-plus": {
811
+ "status": "not_started",
812
+ "skip_reason": null,
813
+ "matched_tasks": []
814
+ },
815
+ "mmlu_prox": {
816
+ "status": "not_started",
817
+ "skip_reason": null,
818
+ "matched_tasks": []
819
+ },
820
+ "mmlu-redux": {
821
+ "status": "not_started",
822
+ "skip_reason": null,
823
+ "matched_tasks": []
824
+ },
825
+ "mmlu-redux-spanish": {
826
+ "status": "not_started",
827
+ "skip_reason": null,
828
+ "matched_tasks": []
829
+ },
830
+ "mmlusr": {
831
+ "status": "not_started",
832
+ "skip_reason": null,
833
+ "matched_tasks": []
834
+ },
835
+ "mmmu": {
836
+ "status": "not_started",
837
+ "skip_reason": null,
838
+ "matched_tasks": []
839
+ },
840
+ "model_written_evals": {
841
+ "status": "not_started",
842
+ "skip_reason": null,
843
+ "matched_tasks": []
844
+ },
845
+ "moral_stories": {
846
+ "status": "not_started",
847
+ "skip_reason": null,
848
+ "matched_tasks": []
849
+ },
850
+ "mts_dialog": {
851
+ "status": "not_started",
852
+ "skip_reason": null,
853
+ "matched_tasks": []
854
+ },
855
+ "multiblimp": {
856
+ "status": "not_started",
857
+ "skip_reason": null,
858
+ "matched_tasks": []
859
+ },
860
+ "mutual": {
861
+ "status": "not_started",
862
+ "skip_reason": null,
863
+ "matched_tasks": []
864
+ },
865
+ "noreval": {
866
+ "status": "skipped",
867
+ "skip_reason": "",
868
+ "matched_tasks": []
869
+ },
870
+ "noticia": {
871
+ "status": "skipped",
872
+ "skip_reason": "",
873
+ "matched_tasks": []
874
+ },
875
+ "nq_open": {
876
+ "status": "not_started",
877
+ "skip_reason": null,
878
+ "matched_tasks": []
879
+ },
880
+ "okapi": {
881
+ "status": "skipped",
882
+ "skip_reason": "",
883
+ "matched_tasks": []
884
+ },
885
+ "olaph": {
886
+ "status": "not_started",
887
+ "skip_reason": null,
888
+ "matched_tasks": []
889
+ },
890
+ "openai-mmmlu": {
891
+ "status": "skipped",
892
+ "skip_reason": "",
893
+ "matched_tasks": []
894
+ },
895
+ "openbookqa": {
896
+ "status": "not_started",
897
+ "skip_reason": null,
898
+ "matched_tasks": []
899
+ },
900
+ "paloma": {
901
+ "status": "skipped",
902
+ "skip_reason": "",
903
+ "matched_tasks": []
904
+ },
905
+ "paws-x": {
906
+ "status": "skipped",
907
+ "skip_reason": "",
908
+ "matched_tasks": []
909
+ },
910
+ "pile": {
911
+ "status": "skipped",
912
+ "skip_reason": "",
913
+ "matched_tasks": []
914
+ },
915
+ "pile_10k": {
916
+ "status": "skipped",
917
+ "skip_reason": "",
918
+ "matched_tasks": []
919
+ },
920
+ "piqa": {
921
+ "status": "evaluated",
922
+ "skip_reason": null,
923
+ "matched_tasks": [
924
+ "piqa"
925
+ ]
926
+ },
927
+ "pisa": {
928
+ "status": "not_started",
929
+ "skip_reason": null,
930
+ "matched_tasks": []
931
+ },
932
+ "polemo2": {
933
+ "status": "skipped",
934
+ "skip_reason": "",
935
+ "matched_tasks": []
936
+ },
937
+ "portuguese_bench": {
938
+ "status": "skipped",
939
+ "skip_reason": "",
940
+ "matched_tasks": []
941
+ },
942
+ "prost": {
943
+ "status": "not_started",
944
+ "skip_reason": null,
945
+ "matched_tasks": []
946
+ },
947
+ "pubmedqa": {
948
+ "status": "not_started",
949
+ "skip_reason": null,
950
+ "matched_tasks": []
951
+ },
952
+ "qa4mre": {
953
+ "status": "not_started",
954
+ "skip_reason": null,
955
+ "matched_tasks": []
956
+ },
957
+ "qasper": {
958
+ "status": "not_started",
959
+ "skip_reason": null,
960
+ "matched_tasks": []
961
+ },
962
+ "race": {
963
+ "status": "not_started",
964
+ "skip_reason": null,
965
+ "matched_tasks": []
966
+ },
967
+ "realtoxicityprompts": {
968
+ "status": "not_started",
969
+ "skip_reason": null,
970
+ "matched_tasks": []
971
+ },
972
+ "ruler": {
973
+ "status": "not_started",
974
+ "skip_reason": null,
975
+ "matched_tasks": []
976
+ },
977
+ "sciq": {
978
+ "status": "evaluated",
979
+ "skip_reason": null,
980
+ "matched_tasks": [
981
+ "sciq"
982
+ ]
983
+ },
984
+ "score": {
985
+ "status": "not_started",
986
+ "skip_reason": null,
987
+ "matched_tasks": []
988
+ },
989
+ "scrolls": {
990
+ "status": "not_started",
991
+ "skip_reason": null,
992
+ "matched_tasks": []
993
+ },
994
+ "simple_cooccurrence_bias": {
995
+ "status": "evaluated",
996
+ "skip_reason": null,
997
+ "matched_tasks": [
998
+ "simple_cooccurrence_bias"
999
+ ]
1000
+ },
1001
+ "siqa": {
1002
+ "status": "not_started",
1003
+ "skip_reason": null,
1004
+ "matched_tasks": []
1005
+ },
1006
+ "slr_bench": {
1007
+ "status": "partial",
1008
+ "skip_reason": null,
1009
+ "matched_tasks": [
1010
+ "slr_bench_all"
1011
+ ]
1012
+ },
1013
+ "spanish_bench": {
1014
+ "status": "skipped",
1015
+ "skip_reason": "",
1016
+ "matched_tasks": []
1017
+ },
1018
+ "squad_completion": {
1019
+ "status": "evaluated",
1020
+ "skip_reason": null,
1021
+ "matched_tasks": [
1022
+ "squad_completion"
1023
+ ]
1024
+ },
1025
+ "squadv2": {
1026
+ "status": "evaluated",
1027
+ "skip_reason": null,
1028
+ "matched_tasks": [
1029
+ "squadv2"
1030
+ ]
1031
+ },
1032
+ "storycloze": {
1033
+ "status": "not_started",
1034
+ "skip_reason": null,
1035
+ "matched_tasks": []
1036
+ },
1037
+ "super_glue": {
1038
+ "status": "not_started",
1039
+ "skip_reason": null,
1040
+ "matched_tasks": []
1041
+ },
1042
+ "swag": {
1043
+ "status": "evaluated",
1044
+ "skip_reason": null,
1045
+ "matched_tasks": [
1046
+ "swag"
1047
+ ]
1048
+ },
1049
+ "swde": {
1050
+ "status": "evaluated",
1051
+ "skip_reason": null,
1052
+ "matched_tasks": [
1053
+ "swde"
1054
+ ]
1055
+ },
1056
+ "tinyBenchmarks": {
1057
+ "status": "skipped",
1058
+ "skip_reason": "",
1059
+ "matched_tasks": []
1060
+ },
1061
+ "tmlu": {
1062
+ "status": "skipped",
1063
+ "skip_reason": "",
1064
+ "matched_tasks": []
1065
+ },
1066
+ "tmmluplus": {
1067
+ "status": "skipped",
1068
+ "skip_reason": "",
1069
+ "matched_tasks": []
1070
+ },
1071
+ "toxigen": {
1072
+ "status": "evaluated",
1073
+ "skip_reason": null,
1074
+ "matched_tasks": [
1075
+ "toxigen"
1076
+ ]
1077
+ },
1078
+ "translation": {
1079
+ "status": "not_started",
1080
+ "skip_reason": null,
1081
+ "matched_tasks": []
1082
+ },
1083
+ "triviaqa": {
1084
+ "status": "evaluated",
1085
+ "skip_reason": null,
1086
+ "matched_tasks": [
1087
+ "triviaqa"
1088
+ ]
1089
+ },
1090
+ "truthfulqa": {
1091
+ "status": "not_started",
1092
+ "skip_reason": null,
1093
+ "matched_tasks": []
1094
+ },
1095
+ "truthfulqa-multi": {
1096
+ "status": "partial",
1097
+ "skip_reason": null,
1098
+ "matched_tasks": [
1099
+ "truthfulqa-multi_mc2_en"
1100
+ ]
1101
+ },
1102
+ "turblimp": {
1103
+ "status": "skipped",
1104
+ "skip_reason": "",
1105
+ "matched_tasks": []
1106
+ },
1107
+ "turkishmmlu": {
1108
+ "status": "skipped",
1109
+ "skip_reason": "",
1110
+ "matched_tasks": []
1111
+ },
1112
+ "ulqa": {
1113
+ "status": "not_started",
1114
+ "skip_reason": null,
1115
+ "matched_tasks": []
1116
+ },
1117
+ "unitxt": {
1118
+ "status": "not_started",
1119
+ "skip_reason": null,
1120
+ "matched_tasks": []
1121
+ },
1122
+ "unscramble": {
1123
+ "status": "not_started",
1124
+ "skip_reason": null,
1125
+ "matched_tasks": []
1126
+ },
1127
+ "webqs": {
1128
+ "status": "not_started",
1129
+ "skip_reason": null,
1130
+ "matched_tasks": []
1131
+ },
1132
+ "wikitext": {
1133
+ "status": "skipped",
1134
+ "skip_reason": "",
1135
+ "matched_tasks": []
1136
+ },
1137
+ "winogender": {
1138
+ "status": "partial",
1139
+ "skip_reason": null,
1140
+ "matched_tasks": [
1141
+ "winogender_all",
1142
+ "winogender_female",
1143
+ "winogender_gotcha",
1144
+ "winogender_gotcha_female",
1145
+ "winogender_gotcha_male",
1146
+ "winogender_male",
1147
+ "winogender_neutral"
1148
+ ]
1149
+ },
1150
+ "winogrande": {
1151
+ "status": "not_started",
1152
+ "skip_reason": null,
1153
+ "matched_tasks": []
1154
+ },
1155
+ "wmdp": {
1156
+ "status": "evaluated",
1157
+ "skip_reason": null,
1158
+ "matched_tasks": [
1159
+ "wmdp",
1160
+ "wmdp_bio",
1161
+ "wmdp_chem",
1162
+ "wmdp_cyber"
1163
+ ]
1164
+ },
1165
+ "wmt2016": {
1166
+ "status": "not_started",
1167
+ "skip_reason": null,
1168
+ "matched_tasks": []
1169
+ },
1170
+ "wsc273": {
1171
+ "status": "not_started",
1172
+ "skip_reason": null,
1173
+ "matched_tasks": []
1174
+ },
1175
+ "xcopa": {
1176
+ "status": "skipped",
1177
+ "skip_reason": "",
1178
+ "matched_tasks": []
1179
+ },
1180
+ "xnli": {
1181
+ "status": "partial",
1182
+ "skip_reason": null,
1183
+ "matched_tasks": [
1184
+ "xnli_en"
1185
+ ]
1186
+ },
1187
+ "xnli_eu": {
1188
+ "status": "skipped",
1189
+ "skip_reason": "",
1190
+ "matched_tasks": []
1191
+ },
1192
+ "xquad": {
1193
+ "status": "partial",
1194
+ "skip_reason": null,
1195
+ "matched_tasks": [
1196
+ "xquad_en"
1197
+ ]
1198
+ },
1199
+ "xstorycloze": {
1200
+ "status": "partial",
1201
+ "skip_reason": null,
1202
+ "matched_tasks": [
1203
+ "xstorycloze_en"
1204
+ ]
1205
+ },
1206
+ "xwinograd": {
1207
+ "status": "skipped",
1208
+ "skip_reason": "",
1209
+ "matched_tasks": []
1210
+ },
1211
+ "zhoblimp": {
1212
+ "status": "skipped",
1213
+ "skip_reason": "",
1214
+ "matched_tasks": []
1215
+ }
1216
+ }
1217
+ }