import html import json from pathlib import Path import gradio as gr import matplotlib.pyplot as plt import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns from huggingface_hub import snapshot_download from src.about import ( INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css, custom_js from src.display.utils import ( BENCHMARK_COLS, COLS, EVAL_COLS, AutoEvalColumn, fields, ) from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN from src.populate import get_evaluation_queue_df, get_leaderboard_df plt.switch_backend("Agg") def restart_space(): API.restart_space(repo_id=REPO_ID) ENABLE_REMOTE_DATA_SYNC = False if ENABLE_REMOTE_DATA_SYNC: try: print(EVAL_REQUESTS_PATH) snapshot_download( repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() try: print(EVAL_RESULTS_PATH) snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) ( finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], label="Select Columns to Display:", ), search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"), ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), ColumnFilter( AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)", ), ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True), ], bool_checkboxgroup_label="Hide models", interactive=False, ) def create_score_df(json_path: Path, decimals: int): data = json.loads(json_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if isinstance(sub_items, dict): cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = df[col].apply( lambda v: ("" if pd.isna(v) else (f"{float(v):.{decimals}f}" if isinstance(v, (int, float)) else v)) ) return df def dataframe_height(df: pd.DataFrame): rows = 0 if df is None else int(getattr(df, "shape", (0, 0))[0]) row_px = 40 header_px = 44 padding_px = 96 height = header_px + (rows * row_px) + padding_px return max(320, min(1200, height)) def create_grouped_score_table_html(df: pd.DataFrame): if df is None or df.empty: return '
' fixed_cols = [c for c in ("ID", "Model") if c in df.columns] group_to_cols = {} group_order = [] data_cols_in_order = [] for col in df.columns: if col in fixed_cols: continue if isinstance(col, str) and "\n" in col: group, metric = col.split("\n", 1) else: group, metric = str(col), str(col) if group not in group_to_cols: group_to_cols[group] = [] group_order.append(group) group_to_cols[group].append((metric, col)) data_cols_in_order.append(col) parts = ['
'] parts.append("") if len(fixed_cols) >= 1: parts.append('') if len(fixed_cols) >= 2: parts.append('') for _ in range(len(data_cols_in_order)): parts.append('') parts.append("") parts.append("") parts.append("") for col in fixed_cols: parts.append(f'') merged_groups = set() for group in group_order: metrics = group_to_cols[group] if len(metrics) == 1 and str(metrics[0][0]) == str(group): merged_groups.add(group) parts.append(f'') else: parts.append( f'' ) parts.append("") parts.append("") for group in group_order: if group in merged_groups: continue for metric, _ in group_to_cols[group]: parts.append(f"") parts.append("") parts.append("") parts.append("") col_indices = [df.columns.get_loc(c) for c in (fixed_cols + data_cols_in_order)] for row in df.itertuples(index=False, name=None): parts.append("") for idx in col_indices: value = row[idx] text = "" if pd.isna(value) else str(value) parts.append(f"") parts.append("") parts.append("
{html.escape(str(col))}
{html.escape(str(group))}
{html.escape(str(group))}
{html.escape(str(metric))}
{html.escape(text)}
") return "".join(parts) def create_raw_score_df(): raw_path = Path(__file__).resolve().parent / "src" / "raw_score.json" return create_score_df(raw_path, decimals=2) def create_unweighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "unweighted_z_score.json" return create_score_df(z_path, decimals=4) def create_weighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "weighted_z_score.json" data = json.loads(z_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if not isinstance(sub_items, dict): continue cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): if cleaned_category == "Overall Score": header = "Overall Score" else: header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) overall_col = "Overall Score" if "Overall Score" in df.columns else None if overall_col is None: for c in df.columns: if isinstance(c, str) and c.endswith("\nOverall Score"): overall_col = c break if overall_col is not None: df[overall_col] = pd.to_numeric(df[overall_col], errors="coerce") df = df.sort_values(by=overall_col, ascending=False, kind="mergesort") cols = list(df.columns) fixed = [c for c in ("ID", "Model") if c in cols] rest = [c for c in cols if c not in set(fixed + [overall_col])] df = df[fixed + [overall_col] + rest] score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = pd.to_numeric(df[col], errors="coerce").apply( lambda v: "" if pd.isna(v) else f"{float(v) * 100:.2f}%" ) if "ID" in df.columns: df["ID"] = pd.to_numeric(df["ID"], errors="coerce").astype("Int64") return df def create_weights_table_html(): weights_path = Path(__file__).resolve().parent / "src" / "weights.json" payload = json.loads(weights_path.read_text(encoding="utf-8")) bounds = payload["bounds"] min_row = bounds["min_row"] min_col = bounds["min_col"] rows = payload["rows"] covered = set() spans = {} for m in payload["merges"]: r1, c1, r2, c2 = m["r1"], m["c1"], m["r2"], m["c2"] spans[(r1, c1)] = {"rowspan": r2 - r1 + 1, "colspan": c2 - c1 + 1} for r in range(r1, r2 + 1): for c in range(c1, c2 + 1): if (r, c) != (r1, c1): covered.add((r, c)) parts = ['
'] for r_index, row in enumerate(rows, start=min_row): parts.append("") for c_index, value in enumerate(row, start=min_col): if (r_index, c_index) in covered: continue span = spans.get((r_index, c_index)) attrs = "" if span is not None: attrs = f" rowspan=\"{span['rowspan']}\" colspan=\"{span['colspan']}\"" tag = "th" if r_index == min_row else "td" if r_index != min_row: if isinstance(value, (int, float)): value = f"{float(value) * 100:.2f}%" elif isinstance(value, str): stripped = value.strip() try: value = f"{float(stripped) * 100:.2f}%" except Exception: pass text = "" if value is None else html.escape(str(value)) parts.append(f"<{tag}{attrs}>{text}") parts.append("") parts.append("
") return "".join(parts) def parse_number(value): if value is None or (isinstance(value, float) and pd.isna(value)): return None if isinstance(value, (int, float)): return float(value) text = str(value).strip() if text == "": return None if text.endswith("%"): try: return float(text[:-1]) except Exception: return None try: return float(text) except Exception: return None def shorten_label(text: str, max_len: int = 18): text = "" if text is None else str(text) if len(text) <= max_len: return text return text[: max_len - 1] + "…" def build_metric_choices(raw_df: pd.DataFrame): choices = [("Overall Score", "weighted::Overall Score")] for col in raw_df.columns: if col in ("ID", "Model"): continue if isinstance(col, str) and "\n" in col: group, metric = col.split("\n", 1) label = f"{group} / {metric}" else: label = str(col) choices.append((label, f"raw::{col}")) return choices def build_rank_bar_plot(metric_key: str): if not isinstance(metric_key, str) or "::" not in metric_key: metric_key = "weighted::Overall Score" source, col = metric_key.split("::", 1) if source == "weighted": df = WEIGHTED_Z_DF else: df = RAW_SCORE_DF if df is None or df.empty or col not in df.columns: fig, ax = plt.subplots(figsize=(10, 4)) ax.set_axis_off() return fig series = [] for _, row in df.iterrows(): model = row.get("Model", "") value = parse_number(row.get(col)) if value is None: continue series.append((str(model), float(value))) series.sort(key=lambda x: x[1], reverse=True) labels = [shorten_label(m) for m, _ in series] values = [v for _, v in series] n = len(values) width = min(22, max(10, 0.55 * max(1, n))) fig, ax = plt.subplots(figsize=(width, 5)) ax.bar(range(n), values) ax.set_xticks(range(n)) ax.set_xticklabels(labels, rotation=35, ha="right") title = "Overall Score" if (source == "weighted" and col == "Overall Score") else str(col) ax.set_title(title) ax.margins(x=0.01) fig.tight_layout() return fig RAW_SCORE_DF = create_raw_score_df() UNWEIGHTED_Z_DF = create_unweighted_z_score_df() WEIGHTED_Z_DF = create_weighted_z_score_df() METRIC_CHOICES = build_metric_choices(RAW_SCORE_DF) DEFAULT_METRIC = "weighted::Overall Score" SCORE_TABLE_HEIGHT_CSS = f""" #raw-score-table .score-table-scroll {{ max-height: {dataframe_height(RAW_SCORE_DF)}px; }} #unweighted-z-table .score-table-scroll {{ max-height: {dataframe_height(UNWEIGHTED_Z_DF)}px; }} #weighted-z-table .score-table-scroll {{ max-height: {dataframe_height(WEIGHTED_Z_DF)}px; }} """ demo = gr.Blocks(css=custom_css + SCORE_TABLE_HEIGHT_CSS) with demo: gr.HTML(custom_js) gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("Result", elem_id="result-tab", id=0): with gr.Tabs(elem_classes="tab-buttons") as nested_tabs: with gr.TabItem("Chart"): metric = gr.Dropdown(choices=METRIC_CHOICES, value=DEFAULT_METRIC, label="Metric") chart = gr.Plot(value=build_rank_bar_plot(DEFAULT_METRIC)) metric.change(build_rank_bar_plot, inputs=metric, outputs=chart) with gr.TabItem("Raw Score"): gr.HTML(create_grouped_score_table_html(RAW_SCORE_DF), elem_id="raw-score-table") with gr.TabItem("Weights"): gr.HTML(create_weights_table_html(), elem_id="weights-table") with gr.TabItem("Unweighted Z-score"): gr.HTML(create_grouped_score_table_html(UNWEIGHTED_Z_DF), elem_id="unweighted-z-table") with gr.TabItem("Weighted Z-score"): gr.HTML(create_grouped_score_table_html(WEIGHTED_Z_DF), elem_id="weighted-z-table") with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) scheduler.start() demo.queue(default_concurrency_limit=40).launch()