Johannes Janosovits commited on
Commit
97cf908
Β·
1 Parent(s): 4568249
Files changed (1) hide show
  1. app.py +57 -15
app.py CHANGED
@@ -397,38 +397,80 @@ def load_harness_matrix(force: bool = False) -> tuple[dict, list[str], set[str],
397
  _icache: dict = {"matrix": None, "models": None, "tasks": None}
398
 
399
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
400
  def load_inspect_matrix() -> tuple[dict, list[str], set[str], str | None]:
401
  """Returns (task_matrix, sorted_models, evaluated_task_names, error).
402
- Reads from pre-generated inspect_summary.json."""
 
 
 
 
403
  if _icache["matrix"] is not None:
404
  return _icache["matrix"], _icache["models"], _icache["tasks"], None
405
 
406
- if not INSPECT_SUMMARY.exists():
407
- return {}, [], set(), "inspect_summary.json not found β€” run: python3 inspect_coverage.py"
 
 
408
 
 
409
  try:
410
- data = json.loads(INSPECT_SUMMARY.read_text())
411
  except Exception as e:
412
  return {}, [], set(), str(e)
413
 
 
 
414
  matrix: dict[str, dict] = {}
415
  all_models: set[str] = set()
416
  evaluated: set[str] = set()
417
 
418
- for entry in data.get("entries", []):
419
- task = entry["task"]
420
- model = entry["model"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
421
  all_models.add(model)
422
  evaluated.add(task)
423
- matrix.setdefault(task, {})[model] = {
424
- "value": entry.get("primary_value"),
425
- "metric": entry.get("primary_metric"),
426
- "n": entry.get("n_samples", 0),
427
- "filename": entry["filename"],
428
- }
429
 
430
- _icache.update(matrix=matrix, models=sorted(all_models), tasks=evaluated)
431
- return matrix, sorted(all_models), evaluated, None
 
432
 
433
 
434
  # ── UI ────────────────────────────────────────────────────────────────────────
 
397
  _icache: dict = {"matrix": None, "models": None, "tasks": None}
398
 
399
 
400
+ def _load_inspect_summary() -> dict[str, dict]:
401
+ """Load inspect_summary.json as a filename→metrics lookup."""
402
+ if not INSPECT_SUMMARY.exists():
403
+ return {}
404
+ try:
405
+ data = json.loads(INSPECT_SUMMARY.read_text())
406
+ return {
407
+ e["filename"]: {
408
+ "task": e["task"],
409
+ "model": e["model"],
410
+ "value": e.get("primary_value"),
411
+ "metric": e.get("primary_metric"),
412
+ "n": e.get("n_samples", 0),
413
+ }
414
+ for e in data.get("entries", [])
415
+ }
416
+ except Exception:
417
+ return {}
418
+
419
+
420
  def load_inspect_matrix() -> tuple[dict, list[str], set[str], str | None]:
421
  """Returns (task_matrix, sorted_models, evaluated_task_names, error).
422
+
423
+ Lists .eval files live from HF every call (when cache is empty).
424
+ Uses inspect_summary.json as a metrics cache for known files;
425
+ falls back to header_only for files not yet in the summary.
426
+ """
427
  if _icache["matrix"] is not None:
428
  return _icache["matrix"], _icache["models"], _icache["tasks"], None
429
 
430
+ from inspect_ai.log import read_eval_log
431
+
432
+ # Metrics cache: filename -> {task, model, value, metric, n}
433
+ summary = _load_inspect_summary()
434
 
435
+ api = HfApi(token=HF_TOKEN)
436
  try:
437
+ all_files = sorted(api.list_repo_files(INSPECT_REPO, repo_type="dataset"))
438
  except Exception as e:
439
  return {}, [], set(), str(e)
440
 
441
+ eval_files = [f for f in all_files if f.endswith(".eval")]
442
+
443
  matrix: dict[str, dict] = {}
444
  all_models: set[str] = set()
445
  evaluated: set[str] = set()
446
 
447
+ for fname in eval_files:
448
+ if fname in summary:
449
+ e = summary[fname]
450
+ task, model = e["task"], e["model"]
451
+ entry = {"value": e["value"], "metric": e["metric"],
452
+ "n": e["n"], "filename": fname}
453
+ else:
454
+ # New file not yet in summary: read header only for task/model
455
+ try:
456
+ local = hf_hub_download(INSPECT_REPO, fname,
457
+ repo_type="dataset", token=HF_TOKEN)
458
+ log = read_eval_log(local, header_only=True)
459
+ task = log.eval.task.split("/")[-1]
460
+ model = log.eval.model
461
+ n = log.eval.dataset.samples if log.eval.dataset else 0
462
+ except Exception:
463
+ continue
464
+ entry = {"value": None, "metric": None, "n": n, "filename": fname}
465
+
466
  all_models.add(model)
467
  evaluated.add(task)
468
+ # Latest file (sorted chronologically by filename) wins
469
+ matrix.setdefault(task, {})[model] = entry
 
 
 
 
470
 
471
+ models = sorted(all_models)
472
+ _icache.update(matrix=matrix, models=models, tasks=evaluated)
473
+ return matrix, models, evaluated, None
474
 
475
 
476
  # ── UI ────────────────────────────────────────────────────────────────────────