Johannes Janosovits commited on
Commit
4568249
Β·
1 Parent(s): 96adbdd

added inspect

Browse files
Files changed (4) hide show
  1. app.py +430 -324
  2. inspect_coverage.py +161 -0
  3. inspect_summary.json +0 -0
  4. requirements.txt +1 -0
app.py CHANGED
@@ -1,138 +1,303 @@
1
  import json
2
  import os
3
- from datetime import datetime
4
  from pathlib import Path, PurePosixPath
5
 
6
  from huggingface_hub import HfApi, hf_hub_download
7
  from shiny import App, reactive, render, ui
8
 
9
  import coverage as cov
 
10
 
11
- DATASET_REPO = "MIMIR-AI-ROUTER/harness_evals"
12
- HF_BLOB_URL = f"https://huggingface.co/datasets/{DATASET_REPO}/blob/main"
 
 
13
  HF_TOKEN = os.getenv("HF_TOKEN")
 
 
 
14
 
15
  CSS = """
16
  :root {
17
- --bg: #1a1a2e;
18
- --bg2: #16213e;
19
- --bg3: #0f3460;
20
- --accent: #e94560;
21
- --link: #4a9eff;
22
- --text: #e0e0e0;
23
- --muted: #8888aa;
24
- --border: #2a2a4a;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  }
26
- * { box-sizing: border-box; margin: 0; padding: 0; }
27
- body { background: var(--bg); color: var(--text); font-family: system-ui, sans-serif; }
28
- .wrap { max-width: 1300px; margin: 0 auto; padding: 24px 20px; }
29
- h1 { font-size: 1.6em; border-bottom: 2px solid var(--accent); padding-bottom: 10px; margin-bottom: 6px; }
30
- .sub { color: var(--muted); font-size: .85em; margin-bottom: 20px; }
31
- .sub a { color: var(--link); text-decoration: none; }
32
- .sub a:hover { text-decoration: underline; }
33
- .controls { margin-bottom: 16px; }
34
- .btn { background: var(--link); color: #fff; border: none; padding: 7px 14px;
35
- border-radius: 4px; cursor: pointer; font-size: .85em; }
36
- .btn:hover { opacity: .85; }
37
- table { width: 100%; border-collapse: collapse; }
38
- th { background: var(--bg2); color: var(--muted); font-size: .75em;
39
- text-transform: uppercase; letter-spacing: .06em;
40
- padding: 9px 12px; text-align: left; border-bottom: 2px solid var(--border); }
41
- td { padding: 9px 12px; font-size: .88em; border-bottom: 1px solid var(--border); vertical-align: middle; }
42
- tr.run-row { cursor: pointer; }
43
- tr.run-row:hover td { background: var(--bg2); }
44
- tr.run-row.active td { background: var(--bg3); }
45
- .model { font-family: monospace; font-size: .82em; color: var(--link); font-weight: 600; }
46
- .a { color: var(--link); text-decoration: none; font-size: .82em; }
47
- .a:hover { text-decoration: underline; }
48
- .detail { background: var(--bg2); border-radius: 6px; padding: 16px; margin-top: 20px; }
49
- .detail h3 { font-size: .95em; color: var(--muted); font-weight: 400;
50
- margin-bottom: 14px; padding-bottom: 8px; border-bottom: 1px solid var(--border); }
51
- .detail th { background: var(--bg); }
52
- .muted { color: var(--muted); }
53
- .err { color: var(--accent); background: rgba(233,69,96,.1); padding: 10px 14px;
54
- border-radius: 4px; border-left: 3px solid var(--accent); margin-bottom: 16px; }
55
- .loading { color: var(--muted); padding: 40px; text-align: center; font-style: italic; }
56
- /* tabs */
57
- .nav-tabs { border-bottom: 2px solid var(--border); margin-bottom: 20px; display: flex; gap: 4px; }
58
- .nav-tab { background: none; border: none; color: var(--muted); padding: 8px 16px;
59
- cursor: pointer; font-size: .9em; border-bottom: 2px solid transparent;
60
- margin-bottom: -2px; }
61
- .nav-tab:hover { color: var(--text); }
62
- .nav-tab.active { color: var(--text); border-bottom-color: var(--accent); }
63
- .tab-pane { display: none; }
64
- .tab-pane.active { display: block; }
65
- /* coverage */
66
- .cov-stats { display: flex; gap: 12px; margin-bottom: 20px; flex-wrap: wrap; }
67
- .stat-chip { background: var(--bg2); border-radius: 6px; padding: 10px 16px;
68
- font-size: .85em; border-left: 3px solid var(--border); }
69
- .stat-chip .n { font-size: 1.5em; font-weight: 700; display: block; }
70
- .chip-ev { border-left-color: #4caf50; }
71
- .chip-part { border-left-color: #ff9800; }
72
- .chip-skip { border-left-color: #8888aa; }
73
- .chip-none { border-left-color: var(--accent); }
74
- .badge { display: inline-block; padding: 2px 7px; border-radius: 3px; font-size: .78em;
75
- font-weight: 600; text-transform: uppercase; letter-spacing: .04em; }
76
- .badge-ev { background: rgba(76,175,80,.18); color: #4caf50; }
77
- .badge-part { background: rgba(255,152,0,.18); color: #ff9800; }
78
- .badge-skip { background: rgba(136,136,170,.18); color: #8888aa; }
79
- .badge-none { background: rgba(233,69,96,.12); color: var(--accent); }
80
- .reason { color: var(--muted); font-size: .82em; font-style: italic; }
81
  """
82
 
83
- # ── data loading ──────────────────────────────────────��───────────────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
 
85
- _cache: dict = {"runs": None, "err": None}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
 
87
 
88
- def _names_and_metrics(groups: dict, tasks: list) -> tuple[str, str]:
89
- """Returns (name_str, metrics_str) for the two summary columns."""
90
- # Prefer group-level aggregates
91
- g_names, g_metrics = [], []
92
- for g_name, g_data in groups.items():
93
- if not isinstance(g_data, dict):
 
 
94
  continue
95
- for k, v in g_data.items():
96
- if "," not in k or not isinstance(v, float):
97
- continue
98
  mname = k.split(",")[0]
99
  if not mname.endswith("_stderr"):
100
- g_names.append(g_name)
101
- g_metrics.append(f"{v:.3f}")
102
- break
103
- if g_names:
104
- return "; ".join(g_names), "; ".join(g_metrics)
105
-
106
- if not tasks:
107
- return "β€”", "β€”"
108
-
109
- # Few tasks: name column lists task names, metrics column lists values
110
- if len(tasks) <= 5:
111
- names, metrics = [], []
112
- for t in tasks:
113
- names.append(t["name"])
114
- first_m = next(iter(t["metrics"]), None)
115
- metrics.append(f"{t['metrics'][first_m]['value']:.3f}" if first_m else "β€”")
116
- return "; ".join(names), "; ".join(metrics)
117
-
118
- # Many tasks without a group
119
- first_m = next(iter(tasks[0]["metrics"]), None)
120
- if first_m:
121
- vals = [t["metrics"][first_m]["value"] for t in tasks if first_m in t["metrics"]]
122
- if vals:
123
- return f"({len(tasks)} tasks)", f"avg {first_m}: {sum(vals)/len(vals):.3f}"
124
- return f"({len(tasks)} tasks)", "β€”"
125
 
126
 
127
  def load_runs(force: bool = False) -> tuple[list, str | None]:
128
- if not force and _cache["runs"] is not None:
129
- return _cache["runs"], _cache["err"]
130
 
131
  api = HfApi(token=HF_TOKEN)
132
  try:
133
- all_files = sorted(api.list_repo_files(DATASET_REPO, repo_type="dataset"))
134
  except Exception as e:
135
- _cache.update(runs=[], err=str(e))
136
  return [], str(e)
137
 
138
  result_files = [
@@ -147,67 +312,126 @@ def load_runs(force: bool = False) -> tuple[list, str | None]:
147
  runs = []
148
  for filename in sorted(result_files, reverse=True):
149
  try:
150
- local = hf_hub_download(DATASET_REPO, filename, repo_type="dataset", token=HF_TOKEN)
151
  with open(local) as f:
152
  data = json.load(f)
153
  except Exception:
154
  continue
155
 
156
  p = PurePosixPath(filename)
157
- folder = p.parent.as_posix() # e.g. "Qwen__Qwen2.5-7B-Instruct"
158
  ts = p.name.removeprefix("results_").removesuffix(".json")
159
  model = data.get("model_name", "unknown")
160
- date_ts = data.get("date")
161
- date_str = datetime.fromtimestamp(date_ts).strftime("%Y-%m-%d %H:%M") if date_ts else "β€”"
162
 
163
  tasks = []
164
  for task_name, td in data.get("results", {}).items():
165
  if not isinstance(td, dict):
166
  continue
167
- metrics: dict = {}
168
- for k, v in td.items():
169
- if k in ("name", "alias", "sample_len") or not isinstance(v, (int, float)):
170
- continue
171
- if "," in k:
172
- mname, filt = k.rsplit(",", 1)
173
- if not mname.endswith("_stderr"):
174
- metrics[mname] = {
175
- "value": v,
176
- "stderr": td.get(f"{mname}_stderr,{filt}"),
177
- }
178
  samples_f = f"{folder}/samples_{task_name}_{ts}.jsonl"
179
  tasks.append({
180
  "name": task_name,
181
  "n": td.get("sample_len", 0),
182
- "metrics": metrics,
 
183
  "samples_file": samples_f if samples_f in sample_files else None,
184
  })
185
 
186
- task_names, metrics = _names_and_metrics(data.get("groups", {}), tasks)
187
- runs.append({
188
- "model": model,
189
- "date": date_str,
190
- "filename": filename,
191
- "ts": ts,
192
- "tasks": tasks,
193
- "n_tasks": len(tasks),
194
- "task_names": task_names,
195
- "metrics": metrics,
196
- })
197
 
198
- _cache.update(runs=runs, err=None)
199
  return runs, None
200
 
201
 
202
- # ── UI ────────────────────────────────────────────────────────────────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
203
 
204
- _TAB_JS = """
205
- function switchTab(name) {
206
- document.querySelectorAll('.nav-tab').forEach(t => t.classList.toggle('active', t.dataset.tab === name));
207
- document.querySelectorAll('.tab-pane').forEach(p => p.classList.toggle('active', p.id === 'tab-' + name));
208
- }
209
- function selectRun(i) { Shiny.setInputValue('sel_idx', i, {priority:'event'}); }
210
- """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
211
 
212
  app_ui = ui.page_fluid(
213
  ui.tags.head(ui.tags.style(CSS)),
@@ -216,35 +440,39 @@ app_ui = ui.page_fluid(
216
  ui.h1("LM Eval Results"),
217
  ui.div(
218
  {"class": "sub"},
219
- "Dataset: ",
220
- ui.tags.a(
221
- DATASET_REPO,
222
- href=f"https://huggingface.co/datasets/{DATASET_REPO}",
223
- target="_blank",
224
- ),
225
- ),
226
- ui.div(
227
- {"class": "controls"},
228
- ui.input_action_button("refresh", "β†Ί Refresh", class_="btn"),
229
  ),
230
- # Tab bar
231
  ui.div(
232
- {"class": "nav-tabs"},
233
- ui.tags.button("Results", {"class": "nav-tab active", "data-tab": "results", "onclick": "switchTab('results')"}),
234
- ui.tags.button("Coverage", {"class": "nav-tab", "data-tab": "coverage", "onclick": "switchTab('coverage')"}),
 
 
235
  ),
236
- # Results tab
237
  ui.div(
238
- {"class": "tab-pane active", "id": "tab-results"},
239
- ui.output_ui("runs_table"),
240
- ui.output_ui("task_detail"),
 
 
 
241
  ),
242
- # Coverage tab
243
  ui.div(
244
- {"class": "tab-pane", "id": "tab-coverage"},
245
- ui.output_ui("coverage_table"),
 
 
 
 
246
  ),
247
- ui.tags.script(_TAB_JS),
248
  ),
249
  )
250
 
@@ -252,181 +480,59 @@ app_ui = ui.page_fluid(
252
  # ── server ────────────────────────────────────────────────────────────────────
253
 
254
  def server(input, output, session):
255
- sel = reactive.Value(None)
256
 
257
  @reactive.effect
258
- @reactive.event(input.refresh)
259
- def _on_refresh():
260
- _cache.update(runs=None, err=None)
261
- sel.set(None)
262
 
263
  @reactive.effect
264
- @reactive.event(input.sel_idx)
265
- def _on_select():
266
- idx = input.sel_idx()
267
- sel.set(None if sel() == idx else idx)
268
 
269
  @output
270
  @render.ui
271
- def runs_table():
272
- input.refresh() # reactive dependency
273
- runs, err = load_runs()
274
-
275
  if err:
276
- return ui.div({"class": "err"}, f"Failed to load data: {err}")
277
- if not runs:
278
- return ui.div({"class": "loading"}, "No eval results found in the dataset.")
279
-
280
- rows = []
281
- for i, run in enumerate(runs):
282
- active = "active" if sel() == i else ""
283
- rows.append(
284
- ui.tags.tr(
285
- {"class": f"run-row {active}", "onclick": f"selectRun({i})"},
286
- ui.tags.td(ui.span({"class": "model"}, run["model"])),
287
- ui.tags.td(run["date"]),
288
- ui.tags.td(str(run["n_tasks"])),
289
- ui.tags.td(run["task_names"]),
290
- ui.tags.td(run["metrics"]),
291
- ui.tags.td(
292
- ui.tags.a(
293
- {"class": "a", "href": f"{HF_BLOB_URL}/{run['filename']}",
294
- "target": "_blank", "onclick": "event.stopPropagation()"},
295
- "πŸ“„ results",
296
- )
297
- ),
298
- )
299
- )
300
-
301
- return ui.tags.table(
302
- ui.tags.thead(
303
- ui.tags.tr(
304
- ui.tags.th("Model"),
305
- ui.tags.th("Date"),
306
- ui.tags.th("Tasks"),
307
- ui.tags.th("Name"),
308
- ui.tags.th("Metrics"),
309
- ui.tags.th("File"),
310
- )
311
- ),
312
- ui.tags.tbody(*rows),
313
- )
314
 
315
- @output
316
- @render.ui
317
- def task_detail():
318
- idx = sel()
319
- if idx is None:
320
- return ui.div()
321
- runs, _ = load_runs()
322
- if not runs or idx >= len(runs):
323
- return ui.div()
324
-
325
- run = runs[idx]
326
- task_rows = []
327
- for task in sorted(run["tasks"], key=lambda t: t["name"]):
328
- metric_parts = []
329
- for mname, mdata in task["metrics"].items():
330
- v = mdata["value"]
331
- se = mdata.get("stderr")
332
- metric_parts.append(
333
- f"{mname}: {v:.3f}" + (f" Β±{se:.3f}" if se is not None else "")
334
- )
335
-
336
- samples_cell = ui.span({"class": "muted"}, "β€”")
337
- if task["samples_file"]:
338
- samples_cell = ui.tags.a(
339
- {"class": "a",
340
- "href": f"{HF_BLOB_URL}/{task['samples_file']}",
341
- "target": "_blank"},
342
- "πŸ“„ samples",
343
- )
344
-
345
- task_rows.append(
346
- ui.tags.tr(
347
- ui.tags.td(task["name"]),
348
- ui.tags.td(str(task["n"])),
349
- ui.tags.td("; ".join(metric_parts) or "β€”"),
350
- ui.tags.td(samples_cell),
351
- )
352
- )
353
-
354
- return ui.div(
355
- {"class": "detail"},
356
- ui.tags.h3(f"{run['model']} Β· {run['date']} Β· {run['n_tasks']} tasks"),
357
- ui.tags.table(
358
- ui.tags.thead(
359
- ui.tags.tr(
360
- ui.tags.th("Task"),
361
- ui.tags.th("N Samples"),
362
- ui.tags.th("Metrics"),
363
- ui.tags.th("Samples File"),
364
- )
365
- ),
366
- ui.tags.tbody(*task_rows),
367
- ),
368
  )
369
 
370
  @output
371
  @render.ui
372
- def coverage_table():
373
- input.refresh() # refresh dependency
374
- runs, _ = load_runs()
375
-
376
- # Collect all evaluated task names from loaded runs
377
- evaluated: set[str] = set()
378
- for run in (runs or []):
379
- for task in run["tasks"]:
380
- evaluated.add(task["name"])
381
-
382
- coverage = cov.compute_coverage(evaluated)
383
- stats = cov.summary_stats(coverage)
384
-
385
- _badge = {
386
- "evaluated": ("ev", "Evaluated"),
387
- "partial": ("part", "Partial"),
388
- "skipped": ("skip", "Skipped"),
389
- "not_started": ("none", "Not started"),
390
- }
391
 
392
- stat_chips = ui.div(
393
- {"class": "cov-stats"},
394
- ui.div({"class": "stat-chip chip-ev"}, ui.tags.span({"class": "n"}, str(stats["evaluated"])), "Evaluated"),
395
- ui.div({"class": "stat-chip chip-part"}, ui.tags.span({"class": "n"}, str(stats["partial"])), "Partial"),
396
- ui.div({"class": "stat-chip chip-none"}, ui.tags.span({"class": "n"}, str(stats["not_started"])), "Not started"),
397
- ui.div({"class": "stat-chip chip-skip"}, ui.tags.span({"class": "n"}, str(stats["skipped"])), "Skipped"),
398
- )
399
 
400
- rows = []
401
- for folder in cov.ALL_FOLDERS:
402
- info = coverage[folder]
403
- status = info["status"]
404
- cls, label = _badge[status]
405
- reason = info["skip_reason"] or ""
406
- matched = info["matched_tasks"]
407
-
408
- rows.append(ui.tags.tr(
409
- ui.tags.td(folder),
410
- ui.tags.td(ui.span({"class": f"badge badge-{cls}"}, label)),
411
- ui.tags.td(
412
- ui.span({"class": "reason"}, reason) if reason
413
- else (ui.span({"class": "muted"}, ", ".join(matched[:5]) + ("…" if len(matched) > 5 else "")) if matched
414
- else ui.span({"class": "muted"}, "β€”"))
415
- ),
416
- ))
417
-
418
- return ui.div(
419
- stat_chips,
420
- ui.tags.table(
421
- ui.tags.thead(
422
- ui.tags.tr(
423
- ui.tags.th("Task Folder"),
424
- ui.tags.th("Status"),
425
- ui.tags.th("Notes / Evaluated tasks"),
426
- )
427
- ),
428
- ui.tags.tbody(*rows),
429
- ),
430
  )
431
 
432
 
 
1
  import json
2
  import os
 
3
  from pathlib import Path, PurePosixPath
4
 
5
  from huggingface_hub import HfApi, hf_hub_download
6
  from shiny import App, reactive, render, ui
7
 
8
  import coverage as cov
9
+ import inspect_coverage as icov
10
 
11
+ HARNESS_REPO = "MIMIR-AI-ROUTER/harness_evals"
12
+ INSPECT_REPO = "MIMIR-AI-ROUTER/inspect_evals"
13
+ HF_HARNESS = f"https://huggingface.co/datasets/{HARNESS_REPO}/blob/main"
14
+ HF_INSPECT = f"https://huggingface.co/datasets/{INSPECT_REPO}/blob/main"
15
  HF_TOKEN = os.getenv("HF_TOKEN")
16
+ INSPECT_SUMMARY = Path(__file__).parent / "inspect_summary.json"
17
+
18
+ # ── CSS ───────────────────────────────────────────────────────────────────────
19
 
20
  CSS = """
21
  :root {
22
+ --bg: #1a1a2e; --bg2: #16213e; --bg3: #0f3460;
23
+ --acc: #e94560; --link: #4a9eff;
24
+ --text: #e0e0e0; --muted:#8888aa; --border:#2a2a4a;
25
+ --ev: #1e3a1e; --ev-t: #6fcf6f;
26
+ --part: #2a2010; --part-t:#e0a020;
27
+ --none-t:#555570; --skip-t:#666680;
28
+ }
29
+ * { box-sizing:border-box; margin:0; padding:0; }
30
+ body { background:var(--bg); color:var(--text); font-family:system-ui,sans-serif; }
31
+ .wrap { max-width:1500px; margin:0 auto; padding:20px; }
32
+ h1 { font-size:1.5em; border-bottom:2px solid var(--acc); padding-bottom:8px; margin-bottom:6px; }
33
+ .sub { color:var(--muted); font-size:.82em; margin-bottom:16px; }
34
+ .sub a { color:var(--link); text-decoration:none; }
35
+
36
+ /* top tabs */
37
+ .top-tabs { display:flex; gap:4px; border-bottom:2px solid var(--border); margin-bottom:18px; }
38
+ .top-tab { background:none; border:none; color:var(--muted); padding:8px 18px;
39
+ cursor:pointer; font-size:.9em; border-bottom:2px solid transparent;
40
+ margin-bottom:-2px; }
41
+ .top-tab:hover { color:var(--text); }
42
+ .top-tab.active { color:var(--text); border-bottom-color:var(--acc); }
43
+ .top-pane { display:none; }
44
+ .top-pane.active { display:block; }
45
+
46
+ /* controls */
47
+ .controls { display:flex; align-items:center; gap:12px; margin-bottom:14px; flex-wrap:wrap; }
48
+ .btn { background:var(--link); color:#fff; border:none; padding:6px 13px;
49
+ border-radius:4px; cursor:pointer; font-size:.82em; }
50
+ .btn:hover { opacity:.85; }
51
+ .filter-bar { display:flex; align-items:center; gap:10px; flex-wrap:wrap;
52
+ margin-bottom:10px; }
53
+ .filter-search { background:var(--bg2); color:var(--text); border:1px solid var(--border);
54
+ border-radius:4px; padding:5px 10px; font-size:.82em; width:200px; }
55
+ .filter-label { color:var(--muted); font-size:.8em; }
56
+ .filter-check { color:var(--muted); font-size:.8em; cursor:pointer;
57
+ display:flex; align-items:center; gap:4px; }
58
+
59
+ /* stat chips */
60
+ .chips { display:flex; gap:10px; margin-bottom:14px; flex-wrap:wrap; }
61
+ .chip { background:var(--bg2); border-radius:5px; padding:8px 14px;
62
+ font-size:.82em; border-left:3px solid var(--border); }
63
+ .chip .n { font-size:1.35em; font-weight:700; display:block; }
64
+ .chip-ev { border-left-color:#4caf50; }
65
+ .chip-part { border-left-color:#ff9800; }
66
+ .chip-skip { border-left-color:var(--muted); }
67
+ .chip-none { border-left-color:var(--acc); }
68
+
69
+ /* matrix table */
70
+ .matrix-wrap { overflow-x:auto; max-height:80vh; overflow-y:auto; }
71
+ .matrix-table { border-collapse:collapse; width:100%; }
72
+ .matrix-table th { background:var(--bg2); color:var(--muted); font-size:.72em;
73
+ text-transform:uppercase; letter-spacing:.06em;
74
+ padding:8px 12px; text-align:left; border-bottom:2px solid var(--border);
75
+ position:sticky; top:0; white-space:nowrap; }
76
+ .matrix-table th.c-name-h { position:sticky; top:0; left:0; z-index:3; min-width:200px; }
77
+ .matrix-table th.c-val-h { min-width:120px; text-align:center; }
78
+ .matrix-table td { padding:7px 10px; border-bottom:1px solid var(--border);
79
+ font-size:.84em; }
80
+ .matrix-table td.c-name { position:sticky; left:0; background:var(--bg); z-index:1;
81
+ min-width:200px; font-size:.83em; }
82
+ .matrix-table tbody tr:hover td { background:var(--bg2); }
83
+ .matrix-table tbody tr:hover td.c-name { background:var(--bg2); }
84
+
85
+ /* cells */
86
+ .c-ev { background:var(--ev); text-align:center; }
87
+ .c-part { background:var(--part); text-align:center; }
88
+ .c-none { text-align:center; color:var(--none-t); }
89
+ .c-skip { text-align:center; color:var(--skip-t); font-size:.78em; font-style:italic; }
90
+ .c-na { text-align:center; color:var(--muted); }
91
+ .val { color:var(--ev-t); font-family:monospace; font-weight:600; font-size:.95em; }
92
+ .val-part { color:var(--part-t); font-family:monospace; font-size:.9em; }
93
+ .metric-lbl { color:var(--muted); font-size:.72em; display:block; }
94
+ .n-lbl { color:var(--muted); font-size:.72em; display:block; }
95
+ .a { color:inherit; text-decoration:none; }
96
+ .a:hover { text-decoration:underline; }
97
+ .badge-skip { font-size:.72em; color:var(--skip-t); }
98
+ .err { color:var(--acc); background:rgba(233,69,96,.1); padding:10px 14px;
99
+ border-radius:4px; border-left:3px solid var(--acc); margin-bottom:12px; }
100
+ .loading { color:var(--muted); padding:30px; text-align:center; font-style:italic; }
101
+ """
102
+
103
+ _JS = """
104
+ function switchTop(name) {
105
+ ['harness','inspect'].forEach(n => {
106
+ document.getElementById('tab-' + n).classList.toggle('active', n === name);
107
+ document.getElementById('pane-' + n).classList.toggle('active', n === name);
108
+ });
109
+ }
110
+ function filterTable(tid) {
111
+ const q = (document.getElementById(tid+'-q') || {value:''}).value.toLowerCase();
112
+ const checks = {
113
+ evaluated: (document.getElementById(tid+'-ev') || {checked:true}).checked,
114
+ partial: (document.getElementById(tid+'-part') || {checked:true}).checked,
115
+ not_started:(document.getElementById(tid+'-none')|| {checked:false}).checked,
116
+ skipped: (document.getElementById(tid+'-skip') || {checked:false}).checked,
117
+ };
118
+ document.querySelectorAll('#' + tid + ' tbody tr').forEach(r => {
119
+ const ok = (!q || r.dataset.name.includes(q)) && (checks[r.dataset.status] !== false);
120
+ r.style.display = ok ? '' : 'none';
121
+ });
122
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123
  """
124
 
125
+ # ── helpers ────────────────────────────────────────────────────────────────────
126
+
127
+ def _short_model(m: str) -> str:
128
+ parts = m.split("/")
129
+ base = parts[-1]
130
+ if len(parts) >= 3:
131
+ return f"{base} ({parts[0]})"
132
+ return base
133
+
134
+
135
+ def _cell_ev(entry: dict, blob_url: str, partial: bool = False) -> ui.Tag:
136
+ val = entry["value"]
137
+ metric = (entry.get("metric") or "").split("/")[-1]
138
+ n = entry.get("n", 0)
139
+ href = f"{blob_url}/{entry['filename']}"
140
+ cls = "c-part" if partial else "c-ev"
141
+ vcls = "val-part" if partial else "val"
142
+ return ui.tags.td(
143
+ {"class": cls},
144
+ ui.tags.a(
145
+ {"class": "a", "href": href, "target": "_blank"},
146
+ ui.span({"class": vcls}, f"{val:.3f}"),
147
+ ),
148
+ ui.span({"class": "metric-lbl"}, metric),
149
+ )
150
+
151
+
152
+ def _cell_na(entry: dict, blob_url: str) -> ui.Tag:
153
+ href = f"{blob_url}/{entry['filename']}"
154
+ return ui.tags.td(
155
+ {"class": "c-na"},
156
+ ui.tags.a({"class": "a", "href": href, "target": "_blank"}, "n/a"),
157
+ )
158
+
159
+
160
+ def _cell_none() -> ui.Tag:
161
+ return ui.tags.td({"class": "c-none"}, "β€”")
162
+
163
+
164
+ def _cell_skip(reason: str) -> ui.Tag:
165
+ return ui.tags.td({"class": "c-skip"}, reason or "skip")
166
+
167
+
168
+ def _render_matrix(
169
+ matrix: dict,
170
+ all_rows: list[str],
171
+ models: list[str],
172
+ cov_data: dict,
173
+ blob_url: str,
174
+ table_id: str,
175
+ show_partial: bool = True,
176
+ ) -> ui.Tag:
177
+ """Render a complete matrix table with filter bar and stat chips."""
178
+
179
+ stats = {"evaluated": 0, "partial": 0, "not_started": 0, "skipped": 0}
180
+ for info in cov_data.values():
181
+ s = info["status"]
182
+ if s in stats:
183
+ stats[s] += 1
184
+
185
+ chips = ui.div(
186
+ {"class": "chips"},
187
+ ui.div({"class": "chip chip-ev"}, ui.span({"class": "n"}, str(stats["evaluated"])), "Evaluated"),
188
+ *([] if not show_partial else [
189
+ ui.div({"class": "chip chip-part"}, ui.span({"class": "n"}, str(stats["partial"])), "Partial"),
190
+ ]),
191
+ ui.div({"class": "chip chip-none"}, ui.span({"class": "n"}, str(stats["not_started"])), "Not started"),
192
+ ui.div({"class": "chip chip-skip"}, ui.span({"class": "n"}, str(stats["skipped"])), "Skipped"),
193
+ )
194
+
195
+ part_check = [] if not show_partial else [
196
+ ui.tags.label(
197
+ {"class": "filter-check"},
198
+ ui.tags.input({"type": "checkbox", "checked": "", "id": table_id + "-part",
199
+ "onchange": f"filterTable('{table_id}')"}),
200
+ " Partial",
201
+ )
202
+ ]
203
+
204
+ filter_bar = ui.div(
205
+ {"class": "filter-bar"},
206
+ ui.tags.input({"type": "text", "class": "filter-search", "placeholder": "Filter by name…",
207
+ "id": table_id + "-q", "oninput": f"filterTable('{table_id}')"}),
208
+ ui.span({"class": "filter-label"}, "Show:"),
209
+ ui.tags.label({"class": "filter-check"},
210
+ ui.tags.input({"type": "checkbox", "checked": "", "id": table_id + "-ev",
211
+ "onchange": f"filterTable('{table_id}')"}), " Evaluated"),
212
+ *part_check,
213
+ ui.tags.label({"class": "filter-check"},
214
+ ui.tags.input({"type": "checkbox", "id": table_id + "-none",
215
+ "onchange": f"filterTable('{table_id}')"}), " Not started"),
216
+ ui.tags.label({"class": "filter-check"},
217
+ ui.tags.input({"type": "checkbox", "id": table_id + "-skip",
218
+ "onchange": f"filterTable('{table_id}')"}), " Skipped"),
219
+ )
220
+
221
+ header = ui.tags.thead(
222
+ ui.tags.tr(
223
+ ui.tags.th("Eval", {"class": "c-name-h"}),
224
+ *[ui.tags.th(_short_model(m), {"class": "c-val-h"}) for m in models],
225
+ )
226
+ )
227
+
228
+ rows = []
229
+ for row_name in all_rows:
230
+ info = cov_data.get(row_name, {"status": "not_started", "skip_reason": None})
231
+ status = info["status"]
232
+ skip_reason = info.get("skip_reason")
233
+
234
+ # n_samples: take from first available model entry
235
+ n = 0
236
+ for m in models:
237
+ e = matrix.get(row_name, {}).get(m)
238
+ if e and e.get("n"):
239
+ n = e["n"]
240
+ break
241
 
242
+ name_cell = ui.tags.td(
243
+ {"class": "c-name"},
244
+ row_name,
245
+ *([] if not n else [ui.tags.br(), ui.span({"class": "n-lbl"}, f"n={n:,}")]),
246
+ )
247
+
248
+ cells: list[ui.Tag] = [name_cell]
249
+ for model in models:
250
+ entry = matrix.get(row_name, {}).get(model)
251
+ if skip_reason is not None:
252
+ cells.append(_cell_skip(skip_reason))
253
+ elif entry is None:
254
+ cells.append(_cell_none())
255
+ elif entry.get("value") is None:
256
+ cells.append(_cell_na(entry, blob_url))
257
+ else:
258
+ cells.append(_cell_ev(entry, blob_url, partial=(status == "partial")))
259
+
260
+ rows.append(ui.tags.tr({"data-name": row_name, "data-status": status}, *cells))
261
+
262
+ return ui.div(
263
+ chips,
264
+ filter_bar,
265
+ ui.div(
266
+ {"class": "matrix-wrap"},
267
+ ui.tags.table(
268
+ {"class": "matrix-table", "id": table_id},
269
+ header,
270
+ ui.tags.tbody(*rows),
271
+ ),
272
+ ),
273
+ )
274
 
275
 
276
+ # ── harness data ──────────────────────────────────────────────────────────────
277
+
278
+ _hcache: dict = {"runs": None, "err": None}
279
+
280
+
281
+ def _parse_harness_task_metrics(td: dict) -> tuple[str | None, float | None]:
282
+ for k, v in td.items():
283
+ if k in ("name", "alias", "sample_len") or not isinstance(v, (int, float)):
284
  continue
285
+ if "," in k:
 
 
286
  mname = k.split(",")[0]
287
  if not mname.endswith("_stderr"):
288
+ return mname, v
289
+ return None, None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
 
291
 
292
  def load_runs(force: bool = False) -> tuple[list, str | None]:
293
+ if not force and _hcache["runs"] is not None:
294
+ return _hcache["runs"], _hcache["err"]
295
 
296
  api = HfApi(token=HF_TOKEN)
297
  try:
298
+ all_files = sorted(api.list_repo_files(HARNESS_REPO, repo_type="dataset"))
299
  except Exception as e:
300
+ _hcache.update(runs=[], err=str(e))
301
  return [], str(e)
302
 
303
  result_files = [
 
312
  runs = []
313
  for filename in sorted(result_files, reverse=True):
314
  try:
315
+ local = hf_hub_download(HARNESS_REPO, filename, repo_type="dataset", token=HF_TOKEN)
316
  with open(local) as f:
317
  data = json.load(f)
318
  except Exception:
319
  continue
320
 
321
  p = PurePosixPath(filename)
322
+ folder = p.parent.as_posix()
323
  ts = p.name.removeprefix("results_").removesuffix(".json")
324
  model = data.get("model_name", "unknown")
 
 
325
 
326
  tasks = []
327
  for task_name, td in data.get("results", {}).items():
328
  if not isinstance(td, dict):
329
  continue
330
+ mname, mval = _parse_harness_task_metrics(td)
 
 
 
 
 
 
 
 
 
 
331
  samples_f = f"{folder}/samples_{task_name}_{ts}.jsonl"
332
  tasks.append({
333
  "name": task_name,
334
  "n": td.get("sample_len", 0),
335
+ "metric": mname,
336
+ "value": mval,
337
  "samples_file": samples_f if samples_f in sample_files else None,
338
  })
339
 
340
+ runs.append({"model": model, "filename": filename, "tasks": tasks})
 
 
 
 
 
 
 
 
 
 
341
 
342
+ _hcache.update(runs=runs, err=None)
343
  return runs, None
344
 
345
 
346
+ def load_harness_matrix(force: bool = False) -> tuple[dict, list[str], set[str], str | None]:
347
+ """Returns (folder_matrix, sorted_models, evaluated_task_names, error)."""
348
+ runs, err = load_runs(force)
349
+ if err:
350
+ return {}, [], set(), err
351
+
352
+ folder_set = set(cov.ALL_FOLDERS)
353
+ evaluated_tasks: set[str] = set()
354
+
355
+ # folder -> model -> list of task entries
356
+ raw: dict[str, dict[str, list]] = {}
357
+ for run in runs:
358
+ model = run["model"]
359
+ for task in run["tasks"]:
360
+ if task["value"] is None:
361
+ continue
362
+ evaluated_tasks.add(task["name"])
363
+ folder = cov._folder_for_task(task["name"], folder_set)
364
+ if folder is None:
365
+ continue
366
+ raw.setdefault(folder, {}).setdefault(model, []).append({
367
+ "task": task["name"],
368
+ "value": task["value"],
369
+ "metric": task["metric"],
370
+ "n": task["n"],
371
+ "filename": run["filename"],
372
+ "is_group": task["name"] == folder,
373
+ })
374
+
375
+ matrix: dict[str, dict] = {f: {} for f in cov.ALL_FOLDERS}
376
+ all_models: set[str] = set()
377
+
378
+ for folder, by_model in raw.items():
379
+ for model, entries in by_model.items():
380
+ all_models.add(model)
381
+ group = [e for e in entries if e["is_group"]]
382
+ best = group[-1] if group else entries[-1]
383
+ is_partial = not bool(group)
384
+ matrix[folder][model] = {
385
+ "value": best["value"],
386
+ "metric": best["metric"],
387
+ "n": best["n"],
388
+ "filename": best["filename"],
389
+ "partial": is_partial,
390
+ }
391
+
392
+ return matrix, sorted(all_models), evaluated_tasks, None
393
 
394
+
395
+ # ── inspect data ──────────────────────────────────────────────────────────────
396
+
397
+ _icache: dict = {"matrix": None, "models": None, "tasks": None}
398
+
399
+
400
+ def load_inspect_matrix() -> tuple[dict, list[str], set[str], str | None]:
401
+ """Returns (task_matrix, sorted_models, evaluated_task_names, error).
402
+ Reads from pre-generated inspect_summary.json."""
403
+ if _icache["matrix"] is not None:
404
+ return _icache["matrix"], _icache["models"], _icache["tasks"], None
405
+
406
+ if not INSPECT_SUMMARY.exists():
407
+ return {}, [], set(), "inspect_summary.json not found β€” run: python3 inspect_coverage.py"
408
+
409
+ try:
410
+ data = json.loads(INSPECT_SUMMARY.read_text())
411
+ except Exception as e:
412
+ return {}, [], set(), str(e)
413
+
414
+ matrix: dict[str, dict] = {}
415
+ all_models: set[str] = set()
416
+ evaluated: set[str] = set()
417
+
418
+ for entry in data.get("entries", []):
419
+ task = entry["task"]
420
+ model = entry["model"]
421
+ all_models.add(model)
422
+ evaluated.add(task)
423
+ matrix.setdefault(task, {})[model] = {
424
+ "value": entry.get("primary_value"),
425
+ "metric": entry.get("primary_metric"),
426
+ "n": entry.get("n_samples", 0),
427
+ "filename": entry["filename"],
428
+ }
429
+
430
+ _icache.update(matrix=matrix, models=sorted(all_models), tasks=evaluated)
431
+ return matrix, sorted(all_models), evaluated, None
432
+
433
+
434
+ # ── UI ────────────────────────────────────────────────────────────────────────
435
 
436
  app_ui = ui.page_fluid(
437
  ui.tags.head(ui.tags.style(CSS)),
 
440
  ui.h1("LM Eval Results"),
441
  ui.div(
442
  {"class": "sub"},
443
+ "Harness: ",
444
+ ui.tags.a(HARNESS_REPO,
445
+ href=f"https://huggingface.co/datasets/{HARNESS_REPO}",
446
+ target="_blank"),
447
+ " Β· Inspect: ",
448
+ ui.tags.a(INSPECT_REPO,
449
+ href=f"https://huggingface.co/datasets/{INSPECT_REPO}",
450
+ target="_blank"),
 
 
451
  ),
 
452
  ui.div(
453
+ {"class": "top-tabs"},
454
+ ui.tags.button("Harness", {"class": "top-tab active", "id": "tab-harness",
455
+ "onclick": "switchTop('harness')"}),
456
+ ui.tags.button("Inspect", {"class": "top-tab", "id": "tab-inspect",
457
+ "onclick": "switchTop('inspect')"}),
458
  ),
 
459
  ui.div(
460
+ {"class": "top-pane active", "id": "pane-harness"},
461
+ ui.div(
462
+ {"class": "controls"},
463
+ ui.input_action_button("refresh_h", "β†Ί Refresh", class_="btn"),
464
+ ),
465
+ ui.output_ui("harness_matrix"),
466
  ),
 
467
  ui.div(
468
+ {"class": "top-pane", "id": "pane-inspect"},
469
+ ui.div(
470
+ {"class": "controls"},
471
+ ui.input_action_button("refresh_i", "β†Ί Refresh", class_="btn"),
472
+ ),
473
+ ui.output_ui("inspect_matrix"),
474
  ),
475
+ ui.tags.script(_JS),
476
  ),
477
  )
478
 
 
480
  # ── server ────────────────────────────────────────────────────────────────────
481
 
482
  def server(input, output, session):
 
483
 
484
  @reactive.effect
485
+ @reactive.event(input.refresh_h)
486
+ def _ref_h():
487
+ _hcache.update(runs=None, err=None)
 
488
 
489
  @reactive.effect
490
+ @reactive.event(input.refresh_i)
491
+ def _ref_i():
492
+ _icache.update(matrix=None, models=None, tasks=None)
 
493
 
494
  @output
495
  @render.ui
496
+ def harness_matrix():
497
+ input.refresh_h()
498
+ matrix, models, ev_tasks, err = load_harness_matrix()
 
499
  if err:
500
+ return ui.div({"class": "err"}, f"Harness load error: {err}")
501
+ if not models:
502
+ return ui.div({"class": "loading"}, "No harness eval results found.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
503
 
504
+ # Override coverage status to include partial from matrix
505
+ coverage = cov.compute_coverage(ev_tasks)
506
+ for folder in cov.ALL_FOLDERS:
507
+ if coverage[folder]["status"] == "evaluated" and matrix.get(folder):
508
+ for m_data in matrix[folder].values():
509
+ if m_data.get("partial"):
510
+ coverage[folder]["status"] = "partial"
511
+ break
512
+
513
+ return _render_matrix(
514
+ matrix, cov.ALL_FOLDERS, models, coverage, HF_HARNESS, "h-mat",
515
+ show_partial=True,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
516
  )
517
 
518
  @output
519
  @render.ui
520
+ def inspect_matrix():
521
+ input.refresh_i()
522
+ matrix, models, ev_tasks, err = load_inspect_matrix()
523
+ if err:
524
+ return ui.div({"class": "err"}, f"Inspect load error: {err}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
525
 
526
+ coverage = icov.compute_coverage(ev_tasks)
527
+ # Rows = all inspect modules + any evaluated tasks not mapped to a module
528
+ unmapped = sorted(ev_tasks - {
529
+ t for m_tasks in coverage.values() for t in m_tasks["matched_tasks"]
530
+ })
531
+ all_rows = icov.ALL_MODULES + unmapped
 
532
 
533
+ return _render_matrix(
534
+ matrix, all_rows, models, coverage, HF_INSPECT, "i-mat",
535
+ show_partial=False,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
536
  )
537
 
538
 
inspect_coverage.py ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Inspect AI eval coverage against the inspect_evals package modules."""
2
+
3
+ from __future__ import annotations
4
+
5
+ # All task/eval modules in inspect_evals (excluding utility modules:
6
+ # constants, utils, metadata, hf_dataset_script_helper).
7
+ ALL_MODULES: list[str] = [
8
+ "abstention_bench", "agent_bench", "agentdojo", "agentharm",
9
+ "agentic_misalignment", "agieval", "ahb", "aime2024", "aime2025",
10
+ "aime2026", "air_bench", "ape", "apps", "arc", "assistant_bench",
11
+ "b3", "bbeh", "bbh", "bbq", "bfcl", "bigcodebench", "bold", "boolq",
12
+ "browse_comp", "chembench", "class_eval", "coconot", "commonsense_qa",
13
+ "compute_eval", "core_bench", "cti_realm", "cve_bench", "cybench",
14
+ "cybergym", "cybermetric", "cyberseceval_2", "cyberseceval_3",
15
+ "docvqa", "drop", "ds1000", "fortress", "frontier_cs", "frontierscience",
16
+ "gaia", "gdm_in_house_ctf", "gdm_intercode_ctf", "gdm_self_proliferation",
17
+ "gdm_self_reasoning", "gdm_stealth", "gdpval", "gpqa", "gsm8k",
18
+ "healthbench", "hellaswag", "hle", "humaneval", "ifeval", "ifevalcode",
19
+ "infinite_bench", "instrumentaleval", "ipi_coding_agent", "kernelbench",
20
+ "lab_bench", "lingoly", "livebench", "livecodebench_pro", "make_me_pay",
21
+ "makemesay", "mask", "math", "mathvista", "mbpp", "medqa", "mgsm",
22
+ "mind2web", "mind2web_sc", "mle_bench", "mlrc_bench", "mmiu", "mmlu",
23
+ "mmlu_pro", "mmmu", "moru", "musr", "niah", "novelty_bench", "onet",
24
+ "osworld", "paperbench", "paws", "persistbench", "personality", "piqa",
25
+ "pre_flight", "pubmedqa", "race_h", "sad", "scbench", "scicode",
26
+ "sciknoweval", "sec_qa", "sevenllm", "simpleqa", "sosbench", "squad",
27
+ "stereoset", "strong_reject", "swe_bench", "swe_lancer", "sycophancy",
28
+ "tac", "tau2", "theagentcompany", "threecb", "truthfulqa", "uccb",
29
+ "usaco", "vimgolf_challenges", "vqa_rad", "vstar_bench", "winogrande",
30
+ "wmdp", "worldsense", "writingbench", "xstest", "zerobench",
31
+ ]
32
+
33
+ # Intentionally skipped modules with reasons (fill in as needed).
34
+ SKIP_LIST: dict[str, str] = {}
35
+
36
+ # Overrides for task names that don't follow the standard module_ prefix.
37
+ # module_name -> task_name_prefix (or exact task name for single-task modules)
38
+ _PREFIX_OVERRIDES: dict[str, str] = {
39
+ "agieval": "agie_",
40
+ "ds1000": "DS-1000",
41
+ }
42
+
43
+
44
+ def module_for_task(task: str, module_set: set[str]) -> str | None:
45
+ """Return the inspect_evals module that owns a given task name."""
46
+ for mod, prefix in _PREFIX_OVERRIDES.items():
47
+ if task == prefix or task.startswith(prefix):
48
+ return mod
49
+ best: str | None = None
50
+ for mod in module_set:
51
+ if task == mod or task.startswith(mod + "_") or task.startswith(mod + "-"):
52
+ if best is None or len(mod) > len(best):
53
+ best = mod
54
+ return best
55
+
56
+
57
+ def compute_coverage(evaluated_tasks: set[str]) -> dict[str, dict]:
58
+ """
59
+ Returns dict: module -> {status, skip_reason, matched_tasks}
60
+ status: "evaluated" | "not_started" | "skipped"
61
+ """
62
+ mod_set = set(ALL_MODULES)
63
+ hits: dict[str, list[str]] = {m: [] for m in ALL_MODULES}
64
+ for task in evaluated_tasks:
65
+ mod = module_for_task(task, mod_set)
66
+ if mod:
67
+ hits[mod].append(task)
68
+
69
+ return {
70
+ mod: {
71
+ "status": "skipped" if mod in SKIP_LIST else ("evaluated" if hits[mod] else "not_started"),
72
+ "skip_reason": SKIP_LIST.get(mod),
73
+ "matched_tasks": sorted(hits[mod]),
74
+ }
75
+ for mod in ALL_MODULES
76
+ }
77
+
78
+
79
+ def summary_stats(coverage: dict[str, dict]) -> dict[str, int]:
80
+ counts: dict[str, int] = {"evaluated": 0, "not_started": 0, "skipped": 0}
81
+ for info in coverage.values():
82
+ counts[info["status"]] += 1
83
+ return counts
84
+
85
+
86
+ # ── CLI: generate inspect_summary.json ───────────────────────────────────────
87
+
88
+ if __name__ == "__main__":
89
+ import json
90
+ import os
91
+ from datetime import datetime, timezone
92
+ from pathlib import Path
93
+
94
+ from huggingface_hub import HfApi, hf_hub_download
95
+ from inspect_ai.log import read_eval_log
96
+
97
+ REPO = "MIMIR-AI-ROUTER/inspect_evals"
98
+ TOKEN = os.getenv("HF_TOKEN")
99
+ _PREFER = ("accuracy", "mean", "correct", "f_score", "jailbreak_rate")
100
+
101
+ def _primary(log):
102
+ if not log.results:
103
+ return None, None
104
+ for s in log.results.scores or []:
105
+ for p in _PREFER:
106
+ if p in (s.metrics or {}) and s.metrics[p].value is not None:
107
+ return f"{s.name}/{p}", s.metrics[p].value
108
+ for s in log.results.scores or []:
109
+ for mk, mv in (s.metrics or {}).items():
110
+ if mv.value is not None:
111
+ return f"{s.name}/{mk}", mv.value
112
+ return None, None
113
+
114
+ print("Fetching inspect eval file list...")
115
+ api = HfApi(token=TOKEN)
116
+ files = sorted(api.list_repo_files(REPO, repo_type="dataset"))
117
+ eval_files = [f for f in files if f.endswith(".eval")]
118
+ print(f" {len(eval_files)} .eval files")
119
+
120
+ entries = []
121
+ for i, fname in enumerate(eval_files, 1):
122
+ print(f" [{i}/{len(eval_files)}] {fname[:70]}")
123
+ local = hf_hub_download(REPO, fname, repo_type="dataset", token=TOKEN)
124
+ log = read_eval_log(local)
125
+
126
+ task = log.eval.task.split("/")[-1]
127
+ model = log.eval.model
128
+ n = log.eval.dataset.samples if log.eval.dataset else 0
129
+
130
+ all_metrics: dict[str, float] = {}
131
+ if log.results:
132
+ for s in log.results.scores or []:
133
+ for mk, mv in (s.metrics or {}).items():
134
+ if mv.value is not None:
135
+ all_metrics[f"{s.name}/{mk}"] = mv.value
136
+
137
+ pm, pv = _primary(log)
138
+ entries.append({
139
+ "task": task,
140
+ "model": model,
141
+ "filename": fname,
142
+ "n_samples": n,
143
+ "metrics": all_metrics,
144
+ "primary_metric": pm,
145
+ "primary_value": pv,
146
+ })
147
+
148
+ out = {
149
+ "generated_at": datetime.now(timezone.utc).isoformat(),
150
+ "entries": entries,
151
+ }
152
+ dest = Path(__file__).parent / "inspect_summary.json"
153
+ dest.write_text(json.dumps(out, indent=2))
154
+ print(f"\nWrote {dest} ({len(entries)} entries)")
155
+
156
+ ev_tasks = {e["task"] for e in entries}
157
+ cov = compute_coverage(ev_tasks)
158
+ stats = summary_stats(cov)
159
+ print(f" evaluated modules : {stats['evaluated']}/{len(ALL_MODULES)}")
160
+ print(f" not started : {stats['not_started']}")
161
+ print(f" skipped : {stats['skipped']}")
inspect_summary.json ADDED
The diff for this file is too large to render. See raw diff
 
requirements.txt CHANGED
@@ -1,2 +1,3 @@
1
  shiny
2
  huggingface_hub
 
 
1
  shiny
2
  huggingface_hub
3
+ inspect-ai