Spaces:
Running
Running
File size: 4,489 Bytes
3c19b11 cdb54bf 78b04f0 3c19b11 78b04f0 3c19b11 7990e30 3c19b11 cdb54bf 3c19b11 3874098 3c19b11 3874098 3c19b11 78b04f0 3874098 7990e30 3c19b11 cdb54bf 3c19b11 cdb54bf 3c19b11 cdb54bf 3c19b11 cdb54bf 3c19b11 cdb54bf 3c19b11 cdb54bf 3c19b11 cdb54bf 3c19b11 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | """Category leaderboard tables and helpers."""
from __future__ import annotations
import html
import pandas as pd
from src.bench_config import (
load_bench_config,
section_order,
taxonomy_label,
taxonomy_order,
)
from src.i18n import get_lang, section_desc_t, section_label_t, t, taxonomy_desc_t
from src.input_modality import harness_input
from src.models import LeaderboardEntry
def section_leaderboard_dataframe(entries: list[LeaderboardEntry], section_id: str) -> pd.DataFrame:
rows = []
for entry in entries:
stats = entry.sections.get(section_id)
rate = stats.success_rate if stats else None
rows.append(
{
"Model": entry.model,
"Harness": entry.harness,
"Input": harness_input(entry.model, entry.harness),
"Tasks": f"{stats.passed}/{stats.total}" if stats else None,
"Success %": round(rate * 100, 2) if rate is not None else None,
}
)
if not rows:
return pd.DataFrame()
frame = pd.DataFrame(rows).sort_values("Success %", ascending=False, na_position="last")
return _localize_columns(frame)
def taxonomy_ids_in_entries(entries: list[LeaderboardEntry]) -> list[str]:
"""UI classes that have at least one scored task in the given runs."""
found: set[str] = set()
for entry in entries:
for class_id, row in (entry.ui_classes or {}).items():
if not isinstance(row, dict):
continue
total = row.get("total")
if isinstance(total, (int, float)) and total > 0:
found.add(str(class_id))
ordered = [class_id for class_id in taxonomy_order() if class_id in found]
return ordered + sorted(found.difference(ordered))
def taxonomy_leaderboard_dataframe(entries: list[LeaderboardEntry], class_id: str) -> pd.DataFrame:
rows = []
for entry in entries:
row = entry.ui_classes.get(class_id) or {}
total = row.get("total")
if not isinstance(total, (int, float)) or total <= 0:
continue
rate = row.get("success_rate")
passed = row.get("passed")
rows.append(
{
"Model": entry.model,
"Harness": entry.harness,
"Input": harness_input(entry.model, entry.harness),
"Tasks": f"{passed}/{int(total)}" if passed is not None else None,
"Success %": round(float(rate) * 100, 2) if isinstance(rate, (int, float)) else None,
}
)
if not rows:
return pd.DataFrame()
frame = pd.DataFrame(rows).sort_values("Success %", ascending=False, na_position="last")
return _localize_columns(frame)
def _localize_columns(frame: pd.DataFrame) -> pd.DataFrame:
if get_lang() != "ru" or frame.empty:
return frame
return frame.rename(
columns={
"Model": t("model"),
"Harness": t("harness"),
"Input": t("input"),
"Tasks": t("tasks_col"),
"Success %": t("success"),
}
)
def category_legend_html(kind: str) -> str:
config = load_bench_config()
if kind == "section":
items = [
(section_label_t(section_id), section_desc_t(section_id))
for section_id in section_order()
]
else:
items = [
(taxonomy_label(class_id), taxonomy_desc_t(class_id))
for class_id in taxonomy_order()
]
spans = "".join(
f'<span class="wab-legend-item" title="{html.escape(desc)}">{html.escape(label)}</span>'
for label, desc in items
)
return f'<div class="wab-category-legend">{spans}</div>'
def category_description_html(kind: str, item_id: str) -> str:
if kind == "section":
desc = section_desc_t(item_id)
label = section_label_t(item_id)
else:
desc = taxonomy_desc_t(item_id)
label = taxonomy_label(item_id)
body = f"<strong>{html.escape(label)}</strong>"
if desc:
body += f" — {html.escape(desc)}"
return f'<p class="wab-category-desc">{body}</p>'
def category_description(kind: str, item_id: str) -> str:
if kind == "section":
desc = section_desc_t(item_id)
label = section_label_t(item_id)
else:
desc = taxonomy_desc_t(item_id)
label = taxonomy_label(item_id)
if not desc:
return f"**{label}**"
return f"**{label}** — {desc}"
|