import html import json from pathlib import Path import gradio as gr import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns from huggingface_hub import snapshot_download from src.about import ( INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css, custom_js from src.display.utils import ( BENCHMARK_COLS, COLS, EVAL_COLS, AutoEvalColumn, fields, ) from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN from src.populate import get_evaluation_queue_df, get_leaderboard_df def restart_space(): API.restart_space(repo_id=REPO_ID) ENABLE_REMOTE_DATA_SYNC = False if ENABLE_REMOTE_DATA_SYNC: try: print(EVAL_REQUESTS_PATH) snapshot_download( repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() try: print(EVAL_RESULTS_PATH) snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN, ) except Exception: restart_space() LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) ( finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], label="Select Columns to Display:", ), search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"), ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), ColumnFilter( AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)", ), ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True), ], bool_checkboxgroup_label="Hide models", interactive=False, ) demo = gr.Blocks(css=custom_css) def create_score_df(json_path: Path, decimals: int): data = json.loads(json_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if isinstance(sub_items, dict): cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = df[col].apply( lambda v: ("" if pd.isna(v) else (f"{float(v):.{decimals}f}" if isinstance(v, (int, float)) else v)) ) return df def dataframe_height(df: pd.DataFrame): rows = 0 if df is None else int(getattr(df, "shape", (0, 0))[0]) row_px = 40 header_px = 44 padding_px = 96 height = header_px + (rows * row_px) + padding_px return max(320, min(1200, height)) def create_raw_score_df(): raw_path = Path(__file__).resolve().parent / "src" / "raw_score.json" return create_score_df(raw_path, decimals=2) def create_unweighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "unweighted_z_score.json" return create_score_df(z_path, decimals=4) def create_weighted_z_score_df(): z_path = Path(__file__).resolve().parent / "src" / "weighted_z_score.json" data = json.loads(z_path.read_text(encoding="utf-8")) if not data: return pd.DataFrame() records = [] for item in data: flat_record = {"ID": item.get("ID"), "Model": item.get("Model")} for category, sub_items in item.items(): if category in ("ID", "Model"): continue if not isinstance(sub_items, dict): continue cleaned_category = str(category).replace("\n", " ") for sub_category, value in sub_items.items(): if cleaned_category == "Overall Score": header = "Overall Score" else: header = f"{cleaned_category}\n{sub_category}" flat_record[header] = value records.append(flat_record) df = pd.DataFrame(records) overall_col = "Overall Score" if "Overall Score" in df.columns else None if overall_col is None: for c in df.columns: if isinstance(c, str) and c.endswith("\nOverall Score"): overall_col = c break if overall_col is not None: df[overall_col] = pd.to_numeric(df[overall_col], errors="coerce") df = df.sort_values(by=overall_col, ascending=False, kind="mergesort") cols = list(df.columns) fixed = [c for c in ("ID", "Model") if c in cols] rest = [c for c in cols if c not in set(fixed + [overall_col])] df = df[fixed + [overall_col] + rest] score_cols = [c for c in df.columns if c not in ("ID", "Model")] for col in score_cols: df[col] = pd.to_numeric(df[col], errors="coerce").apply( lambda v: "" if pd.isna(v) else f"{float(v) * 100:.2f}%" ) if "ID" in df.columns: df["ID"] = pd.to_numeric(df["ID"], errors="coerce").astype("Int64") return df def create_weights_table_html(): weights_path = Path(__file__).resolve().parent / "src" / "weights.json" payload = json.loads(weights_path.read_text(encoding="utf-8")) bounds = payload["bounds"] min_row = bounds["min_row"] min_col = bounds["min_col"] rows = payload["rows"] covered = set() spans = {} for m in payload["merges"]: r1, c1, r2, c2 = m["r1"], m["c1"], m["r2"], m["c2"] spans[(r1, c1)] = {"rowspan": r2 - r1 + 1, "colspan": c2 - c1 + 1} for r in range(r1, r2 + 1): for c in range(c1, c2 + 1): if (r, c) != (r1, c1): covered.add((r, c)) parts = ['