import html import json from pathlib import Path import gradio as gr import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns from huggingface_hub import snapshot_download from src.about import ( INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css, custom_js from src.display.utils import ( BENCHMARK_COLS, COLS, EVAL_COLS, AutoEvalColumn, fields, ) from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN from src.populate import get_evaluation_queue_df, get_leaderboard_df def restart_space(): API.restart_space(repo_id=REPO_ID) ENABLE_REMOTE_DATA_SYNC = False if ENABLE_REMOTE_DATA_SYNC: try: print(EVAL_REQUESTS_PATH) snapshot_download( repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() try: print(EVAL_RESULTS_PATH) snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) ( finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], label="Select Columns to Display:", ), search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"), ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), ColumnFilter( AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)", ), ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True), ], bool_checkboxgroup_label="Hide models", interactive=False, ) demo = gr.Blocks(css=custom_css) def create_score_df(json_path: Path, decimals: int): data = json.loads(json_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if isinstance(sub_items, dict): cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = df[col].apply( lambda v: ("" if pd.isna(v) else (f"{float(v):.{decimals}f}" if isinstance(v, (int, float)) else v)) ) return df def dataframe_height(df: pd.DataFrame): rows = 0 if df is None else int(getattr(df, "shape", (0, 0))[0]) row_px = 40 header_px = 44 padding_px = 96 height = header_px + (rows * row_px) + padding_px return max(320, min(1200, height)) def create_raw_score_df(): raw_path = Path(__file__).resolve().parent / "src" / "raw_score.json" return create_score_df(raw_path, decimals=2) def create_unweighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "unweighted_z_score.json" return create_score_df(z_path, decimals=4) def create_weighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "weighted_z_score.json" data = json.loads(z_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if not isinstance(sub_items, dict): continue cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): if cleaned_category == "Overall Score": header = "Overall Score" else: header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) overall_col = "Overall Score" if "Overall Score" in df.columns else None if overall_col is None: for c in df.columns: if isinstance(c, str) and c.endswith("\nOverall Score"): overall_col = c break if overall_col is not None: df[overall_col] = pd.to_numeric(df[overall_col], errors="coerce") df = df.sort_values(by=overall_col, ascending=False, kind="mergesort") cols = list(df.columns) fixed = [c for c in ("ID", "Model") if c in cols] rest = [c for c in cols if c not in set(fixed + [overall_col])] df = df[fixed + [overall_col] + rest] score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = pd.to_numeric(df[col], errors="coerce").apply( lambda v: "" if pd.isna(v) else f"{float(v) * 100:.2f}%" ) if "ID" in df.columns: df["ID"] = pd.to_numeric(df["ID"], errors="coerce").astype("Int64") return df def create_weights_table_html(): weights_path = Path(__file__).resolve().parent / "src" / "weights.json" payload = json.loads(weights_path.read_text(encoding="utf-8")) bounds = payload["bounds"] min_row = bounds["min_row"] min_col = bounds["min_col"] rows = payload["rows"] covered = set() spans = {} for m in payload["merges"]: r1, c1, r2, c2 = m["r1"], m["c1"], m["r2"], m["c2"] spans[(r1, c1)] = {"rowspan": r2 - r1 + 1, "colspan": c2 - c1 + 1} for r in range(r1, r2 + 1): for c in range(c1, c2 + 1): if (r, c) != (r1, c1): covered.add((r, c)) parts = ['
'] for r_index, row in enumerate(rows, start=min_row): parts.append("") for c_index, value in enumerate(row, start=min_col): if (r_index, c_index) in covered: continue span = spans.get((r_index, c_index)) attrs = "" if span is not None: attrs = f" rowspan=\"{span['rowspan']}\" colspan=\"{span['colspan']}\"" tag = "th" if r_index == min_row else "td" if r_index != min_row: if isinstance(value, (int, float)): value = f"{float(value):.4f}" elif isinstance(value, str): stripped = value.strip() try: value = f"{float(stripped):.4f}" except Exception: pass text = "" if value is None else html.escape(str(value)) parts.append(f"<{tag}{attrs}>{text}") parts.append("") parts.append("
") return "".join(parts) with demo: gr.HTML(custom_js) gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("Result", elem_id="result-tab", id=0): with gr.Tabs(elem_classes="tab-buttons") as nested_tabs: with gr.TabItem("Raw Score"): raw_score_df = create_raw_score_df() column_widths = [40, 220] + [180] * (len(raw_score_df.columns) - 2) gr.DataFrame( raw_score_df, wrap=True, column_widths=column_widths, row_count=(len(raw_score_df), "fixed"), height=dataframe_height(raw_score_df), elem_id="raw-score-table", ) with gr.TabItem("Weights"): gr.HTML(create_weights_table_html(), elem_id="weights-table") with gr.TabItem("Unweighted Z-score"): unweighted_z_df = create_unweighted_z_score_df() column_widths = [40, 220] + [180] * (len(unweighted_z_df.columns) - 2) gr.DataFrame( unweighted_z_df, wrap=True, column_widths=column_widths, row_count=(len(unweighted_z_df), "fixed"), height=dataframe_height(unweighted_z_df), elem_id="unweighted-z-table", ) with gr.TabItem("Weighted Z-score"): weighted_z_df = create_weighted_z_score_df() column_widths = [40, 220] + [180] * (len(weighted_z_df.columns) - 2) gr.DataFrame( weighted_z_df, wrap=True, column_widths=column_widths, row_count=(len(weighted_z_df), "fixed"), height=dataframe_height(weighted_z_df), elem_id="weighted-z-table", ) with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) scheduler.start() demo.queue(default_concurrency_limit=40).launch()