Spaces:
Sleeping
Sleeping
Johannes Janosovits commited on
Commit Β·
97cf908
1
Parent(s): 4568249
evals
Browse files
app.py
CHANGED
|
@@ -397,38 +397,80 @@ def load_harness_matrix(force: bool = False) -> tuple[dict, list[str], set[str],
|
|
| 397 |
_icache: dict = {"matrix": None, "models": None, "tasks": None}
|
| 398 |
|
| 399 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 400 |
def load_inspect_matrix() -> tuple[dict, list[str], set[str], str | None]:
|
| 401 |
"""Returns (task_matrix, sorted_models, evaluated_task_names, error).
|
| 402 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 403 |
if _icache["matrix"] is not None:
|
| 404 |
return _icache["matrix"], _icache["models"], _icache["tasks"], None
|
| 405 |
|
| 406 |
-
|
| 407 |
-
|
|
|
|
|
|
|
| 408 |
|
|
|
|
| 409 |
try:
|
| 410 |
-
|
| 411 |
except Exception as e:
|
| 412 |
return {}, [], set(), str(e)
|
| 413 |
|
|
|
|
|
|
|
| 414 |
matrix: dict[str, dict] = {}
|
| 415 |
all_models: set[str] = set()
|
| 416 |
evaluated: set[str] = set()
|
| 417 |
|
| 418 |
-
for
|
| 419 |
-
|
| 420 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 421 |
all_models.add(model)
|
| 422 |
evaluated.add(task)
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
"metric": entry.get("primary_metric"),
|
| 426 |
-
"n": entry.get("n_samples", 0),
|
| 427 |
-
"filename": entry["filename"],
|
| 428 |
-
}
|
| 429 |
|
| 430 |
-
|
| 431 |
-
|
|
|
|
| 432 |
|
| 433 |
|
| 434 |
# ββ UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 397 |
_icache: dict = {"matrix": None, "models": None, "tasks": None}
|
| 398 |
|
| 399 |
|
| 400 |
+
def _load_inspect_summary() -> dict[str, dict]:
|
| 401 |
+
"""Load inspect_summary.json as a filenameβmetrics lookup."""
|
| 402 |
+
if not INSPECT_SUMMARY.exists():
|
| 403 |
+
return {}
|
| 404 |
+
try:
|
| 405 |
+
data = json.loads(INSPECT_SUMMARY.read_text())
|
| 406 |
+
return {
|
| 407 |
+
e["filename"]: {
|
| 408 |
+
"task": e["task"],
|
| 409 |
+
"model": e["model"],
|
| 410 |
+
"value": e.get("primary_value"),
|
| 411 |
+
"metric": e.get("primary_metric"),
|
| 412 |
+
"n": e.get("n_samples", 0),
|
| 413 |
+
}
|
| 414 |
+
for e in data.get("entries", [])
|
| 415 |
+
}
|
| 416 |
+
except Exception:
|
| 417 |
+
return {}
|
| 418 |
+
|
| 419 |
+
|
| 420 |
def load_inspect_matrix() -> tuple[dict, list[str], set[str], str | None]:
|
| 421 |
"""Returns (task_matrix, sorted_models, evaluated_task_names, error).
|
| 422 |
+
|
| 423 |
+
Lists .eval files live from HF every call (when cache is empty).
|
| 424 |
+
Uses inspect_summary.json as a metrics cache for known files;
|
| 425 |
+
falls back to header_only for files not yet in the summary.
|
| 426 |
+
"""
|
| 427 |
if _icache["matrix"] is not None:
|
| 428 |
return _icache["matrix"], _icache["models"], _icache["tasks"], None
|
| 429 |
|
| 430 |
+
from inspect_ai.log import read_eval_log
|
| 431 |
+
|
| 432 |
+
# Metrics cache: filename -> {task, model, value, metric, n}
|
| 433 |
+
summary = _load_inspect_summary()
|
| 434 |
|
| 435 |
+
api = HfApi(token=HF_TOKEN)
|
| 436 |
try:
|
| 437 |
+
all_files = sorted(api.list_repo_files(INSPECT_REPO, repo_type="dataset"))
|
| 438 |
except Exception as e:
|
| 439 |
return {}, [], set(), str(e)
|
| 440 |
|
| 441 |
+
eval_files = [f for f in all_files if f.endswith(".eval")]
|
| 442 |
+
|
| 443 |
matrix: dict[str, dict] = {}
|
| 444 |
all_models: set[str] = set()
|
| 445 |
evaluated: set[str] = set()
|
| 446 |
|
| 447 |
+
for fname in eval_files:
|
| 448 |
+
if fname in summary:
|
| 449 |
+
e = summary[fname]
|
| 450 |
+
task, model = e["task"], e["model"]
|
| 451 |
+
entry = {"value": e["value"], "metric": e["metric"],
|
| 452 |
+
"n": e["n"], "filename": fname}
|
| 453 |
+
else:
|
| 454 |
+
# New file not yet in summary: read header only for task/model
|
| 455 |
+
try:
|
| 456 |
+
local = hf_hub_download(INSPECT_REPO, fname,
|
| 457 |
+
repo_type="dataset", token=HF_TOKEN)
|
| 458 |
+
log = read_eval_log(local, header_only=True)
|
| 459 |
+
task = log.eval.task.split("/")[-1]
|
| 460 |
+
model = log.eval.model
|
| 461 |
+
n = log.eval.dataset.samples if log.eval.dataset else 0
|
| 462 |
+
except Exception:
|
| 463 |
+
continue
|
| 464 |
+
entry = {"value": None, "metric": None, "n": n, "filename": fname}
|
| 465 |
+
|
| 466 |
all_models.add(model)
|
| 467 |
evaluated.add(task)
|
| 468 |
+
# Latest file (sorted chronologically by filename) wins
|
| 469 |
+
matrix.setdefault(task, {})[model] = entry
|
|
|
|
|
|
|
|
|
|
|
|
|
| 470 |
|
| 471 |
+
models = sorted(all_models)
|
| 472 |
+
_icache.update(matrix=matrix, models=models, tasks=evaluated)
|
| 473 |
+
return matrix, models, evaluated, None
|
| 474 |
|
| 475 |
|
| 476 |
# ββ UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|