"""Category leaderboard tables and helpers.""" from __future__ import annotations import html import pandas as pd from src.bench_config import ( load_bench_config, section_order, taxonomy_label, taxonomy_order, ) from src.i18n import get_lang, section_desc_t, section_label_t, t, taxonomy_desc_t from src.input_modality import harness_input from src.models import LeaderboardEntry def section_leaderboard_dataframe(entries: list[LeaderboardEntry], section_id: str) -> pd.DataFrame: rows = [] for entry in entries: stats = entry.sections.get(section_id) rate = stats.success_rate if stats else None rows.append( { "Model": entry.model, "Harness": entry.harness, "Input": harness_input(entry.model, entry.harness), "Tasks": f"{stats.passed}/{stats.total}" if stats else None, "Success %": round(rate * 100, 2) if rate is not None else None, } ) if not rows: return pd.DataFrame() frame = pd.DataFrame(rows).sort_values("Success %", ascending=False, na_position="last") return _localize_columns(frame) def taxonomy_ids_in_entries(entries: list[LeaderboardEntry]) -> list[str]: """UI classes that have at least one scored task in the given runs.""" found: set[str] = set() for entry in entries: for class_id, row in (entry.ui_classes or {}).items(): if not isinstance(row, dict): continue total = row.get("total") if isinstance(total, (int, float)) and total > 0: found.add(str(class_id)) ordered = [class_id for class_id in taxonomy_order() if class_id in found] return ordered + sorted(found.difference(ordered)) def taxonomy_leaderboard_dataframe(entries: list[LeaderboardEntry], class_id: str) -> pd.DataFrame: rows = [] for entry in entries: row = entry.ui_classes.get(class_id) or {} total = row.get("total") if not isinstance(total, (int, float)) or total <= 0: continue rate = row.get("success_rate") passed = row.get("passed") rows.append( { "Model": entry.model, "Harness": entry.harness, "Input": harness_input(entry.model, entry.harness), "Tasks": f"{passed}/{int(total)}" if passed is not None else None, "Success %": round(float(rate) * 100, 2) if isinstance(rate, (int, float)) else None, } ) if not rows: return pd.DataFrame() frame = pd.DataFrame(rows).sort_values("Success %", ascending=False, na_position="last") return _localize_columns(frame) def _localize_columns(frame: pd.DataFrame) -> pd.DataFrame: if get_lang() != "ru" or frame.empty: return frame return frame.rename( columns={ "Model": t("model"), "Harness": t("harness"), "Input": t("input"), "Tasks": t("tasks_col"), "Success %": t("success"), } ) def category_legend_html(kind: str) -> str: config = load_bench_config() if kind == "section": items = [ (section_label_t(section_id), section_desc_t(section_id)) for section_id in section_order() ] else: items = [ (taxonomy_label(class_id), taxonomy_desc_t(class_id)) for class_id in taxonomy_order() ] spans = "".join( f'{html.escape(label)}' for label, desc in items ) return f'
{body}
' def category_description(kind: str, item_id: str) -> str: if kind == "section": desc = section_desc_t(item_id) label = section_label_t(item_id) else: desc = taxonomy_desc_t(item_id) label = taxonomy_label(item_id) if not desc: return f"**{label}**" return f"**{label}** — {desc}"