import gradio as gr from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler import html from pathlib import Path from huggingface_hub import snapshot_download from src.about import ( CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css from src.display.utils import ( BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, WeightType, Precision ) from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN from src.populate import get_evaluation_queue_df, get_leaderboard_df from src.submission.submit import add_new_eval def restart_space(): API.restart_space(repo_id=REPO_ID) ENABLE_REMOTE_DATA_SYNC = False if ENABLE_REMOTE_DATA_SYNC: try: print(EVAL_REQUESTS_PATH) snapshot_download( repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() try: print(EVAL_RESULTS_PATH) snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) ( finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], label="Select Columns to Display:", ), search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"), ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), ColumnFilter( AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)", ), ColumnFilter( AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True ), ], bool_checkboxgroup_label="Hide models", interactive=False, ) demo = gr.Blocks(css=custom_css) import json def create_score_df(json_path: Path, decimals: int): data = json.loads(json_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if isinstance(sub_items, dict): cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = df[col].apply( lambda v: ( "" if pd.isna(v) else (f"{float(v):.{decimals}f}" if isinstance(v, (int, float)) else v) ) ) return df def create_raw_score_df(): raw_path = Path(__file__).resolve().parent / "src" / "raw_score.json" return create_score_df(raw_path, decimals=2) def create_unweighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "unweighted_z_score.json" return create_score_df(z_path, decimals=4) def create_weighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "weighted_z_score.json" data = json.loads(z_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if not isinstance(sub_items, dict): continue cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): if cleaned_category == "Overall Score": header = "Overall Score" else: header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) overall_col = "Overall Score" if "Overall Score" in df.columns else None if overall_col is None: for c in df.columns: if isinstance(c, str) and c.endswith("\nOverall Score"): overall_col = c break if overall_col is not None: df[overall_col] = pd.to_numeric(df[overall_col], errors="coerce") df = df.sort_values(by=overall_col, ascending=False, kind="mergesort") cols = list(df.columns) fixed = [c for c in ("ID", "Model") if c in cols] rest = [c for c in cols if c not in set(fixed + [overall_col])] df = df[fixed + [overall_col] + rest] score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = pd.to_numeric(df[col], errors="coerce").apply( lambda v: "" if pd.isna(v) else f"{float(v) * 100:.2f}%" ) if "ID" in df.columns: df["ID"] = pd.to_numeric(df["ID"], errors="coerce").astype("Int64") return df def create_weights_table_html(): weights_path = Path(__file__).resolve().parent / "src" / "weights.json" payload = json.loads(weights_path.read_text(encoding="utf-8")) bounds = payload["bounds"] min_row = bounds["min_row"] min_col = bounds["min_col"] rows = payload["rows"] covered = set() spans = {} for m in payload["merges"]: r1, c1, r2, c2 = m["r1"], m["c1"], m["r2"], m["c2"] spans[(r1, c1)] = {"rowspan": r2 - r1 + 1, "colspan": c2 - c1 + 1} for r in range(r1, r2 + 1): for c in range(c1, c2 + 1): if (r, c) != (r1, c1): covered.add((r, c)) parts = ["
" ] for r_index, row in enumerate(rows, start=min_row): parts.append("") for c_index, value in enumerate(row, start=min_col): if (r_index, c_index) in covered: continue span = spans.get((r_index, c_index)) attrs = "" if span is not None: attrs = f" rowspan=\"{span['rowspan']}\" colspan=\"{span['colspan']}\"" tag = "th" if r_index == min_row else "td" if r_index != min_row: if isinstance(value, (int, float)): value = f"{float(value):.4f}" elif isinstance(value, str): stripped = value.strip() try: value = f"{float(stripped):.4f}" except Exception: pass text = "" if value is None else html.escape(str(value)) parts.append(f"<{tag}{attrs}>{text}") parts.append("") parts.append("
") return "".join(parts) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("Result", elem_id="result-tab", id=0): with gr.Tabs(elem_classes="tab-buttons") as nested_tabs: with gr.TabItem("Raw Score"): raw_score_df = create_raw_score_df() column_widths = [40, 220] + [180] * (len(raw_score_df.columns) - 2) gr.DataFrame( raw_score_df, wrap=True, column_widths=column_widths, row_count=(len(raw_score_df), "fixed"), height=700, elem_id="raw-score-table", ) with gr.TabItem("Weights"): gr.HTML(create_weights_table_html(), elem_id="weights-table") with gr.TabItem("Unweighted Z-score"): unweighted_z_df = create_unweighted_z_score_df() column_widths = [40, 220] + [180] * (len(unweighted_z_df.columns) - 2) gr.DataFrame( unweighted_z_df, wrap=True, column_widths=column_widths, row_count=(len(unweighted_z_df), "fixed"), height=700, elem_id="unweighted-z-table", ) with gr.TabItem("Weighted Z-score"): weighted_z_df = create_weighted_z_score_df() column_widths = [40, 220] + [180] * (len(weighted_z_df.columns) - 2) gr.DataFrame( weighted_z_df, wrap=True, column_widths=column_widths, row_count=(len(weighted_z_df), "fixed"), height=700, elem_id="weighted-z-table", ) with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) scheduler.start() demo.queue(default_concurrency_limit=40).launch()