Duplicate from gradio-templates/leaderboard

Co-authored-by: Abubakar Abid <abidlabs@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -0,0 +1,51 @@
+---
+title: Example Leaderboard Template
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+app_file: app.py
+pinned: true
+license: apache-2.0
+short_description: Duplicate this leaderboard to initialize your own!
+sdk_version: 6.26.0
+tags:
+- leaderboard
+---
+
+# Start the configuration
+
+This template uses Gradio 6 Server mode: a custom responsive frontend backed by named Gradio API endpoints. Duplicate the Space, then customize the copy and benchmark definitions below.
+
+Most of the variables to change for a default leaderboard are in `src/envs.py` (replace the paths for your leaderboard) and `src/about.py` (for its title, copy, and tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+- the custom frontend in `static/index.html`, `static/styles.css`, and `static/app.js`

app.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import html
+import logging
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from threading import RLock
+from typing import Any
+
+import gradio as gr
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from huggingface_hub import snapshot_download
+
+from src.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    TITLE,
+    Tasks,
+)
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    ModelType,
+    Precision,
+    WeightType,
+)
+from src.envs import EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+
+logging.basicConfig(level=logging.INFO)
+LOGGER = logging.getLogger("leaderboard")
+
+ROOT = Path(__file__).parent
+STATIC_DIR = ROOT / "static"
+MODEL_LINK_RE = re.compile(r'href=["\']https://huggingface\.co/([^"\']+)["\']')
+DATA_LOCK = RLock()
+DATA: dict[str, Any] = {}
+
+
+def _download_dataset(repo_id: str, local_dir: str) -> None:
+    Path(local_dir).mkdir(parents=True, exist_ok=True)
+    snapshot_download(
+        repo_id=repo_id,
+        local_dir=local_dir,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+        token=TOKEN,
+    )
+
+
+def _json_value(value: Any) -> Any:
+    if pd.isna(value):
+        return None
+    if hasattr(value, "item"):
+        value = value.item()
+    return value
+
+
+def _model_id(value: Any) -> str:
+    text = str(value or "")
+    if match := MODEL_LINK_RE.search(text):
+        return html.unescape(match.group(1))
+    return html.unescape(re.sub("<[^>]+>", "", text))
+
+
+def _records(frame: pd.DataFrame) -> list[dict[str, Any]]:
+    records = []
+    for row in frame.to_dict(orient="records"):
+        clean_row = {str(key): _json_value(value) for key, value in row.items()}
+        if "Model" in clean_row:
+            clean_row["Model"] = _model_id(clean_row["Model"])
+        if "model" in clean_row:
+            clean_row["model"] = _model_id(clean_row["model"])
+        records.append(clean_row)
+    return records
+
+
+def _empty_payload(message: str | None = None) -> dict[str, Any]:
+    return {
+        "rows": [],
+        "queues": {"finished": [], "running": [], "pending": []},
+        "benchmarks": [task.value.col_name for task in Tasks],
+        "model_types": [item.to_str(" · ") for item in ModelType if item is not ModelType.Unknown],
+        "precisions": [item.value.name for item in Precision if item is not Precision.Unknown],
+        "weight_types": [item.value.name for item in WeightType],
+        "copy": {
+            "title": TITLE,
+            "about": LLM_BENCHMARKS_TEXT.strip(),
+            "submission": EVALUATION_QUEUE_TEXT.strip(),
+            "citation_label": CITATION_BUTTON_LABEL,
+            "citation": CITATION_BUTTON_TEXT.strip(),
+        },
+        "updated_at": datetime.now(timezone.utc).isoformat(),
+        "error": message,
+    }
+
+
+def refresh_data(*, download: bool = True) -> dict[str, Any]:
+    """Refresh leaderboard and queue data without taking the Space offline."""
+    errors = []
+    if download:
+        for repo_id, local_dir in (
+            (QUEUE_REPO, EVAL_REQUESTS_PATH),
+            (RESULTS_REPO, EVAL_RESULTS_PATH),
+        ):
+            try:
+                _download_dataset(repo_id, local_dir)
+            except Exception as exc:  # Keep serving the last successful snapshot.
+                LOGGER.exception("Could not refresh %s", repo_id)
+                errors.append(f"Could not refresh {repo_id}: {exc}")
+
+    payload = _empty_payload("; ".join(errors) or None)
+    try:
+        leaderboard = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+        payload["rows"] = _records(leaderboard)
+    except Exception as exc:
+        LOGGER.exception("Could not build leaderboard data")
+        payload["error"] = f"Could not build leaderboard data: {exc}"
+
+    try:
+        finished, running, pending = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+        payload["queues"] = {
+            "finished": _records(finished),
+            "running": _records(running),
+            "pending": _records(pending),
+        }
+    except Exception as exc:
+        LOGGER.exception("Could not build evaluation queues")
+        payload["error"] = f"Could not build evaluation queues: {exc}"
+
+    with DATA_LOCK:
+        DATA.clear()
+        DATA.update(payload)
+        return dict(DATA)
+
+
+server = gr.Server(
+    title="Open LLM Benchmark",
+    description="A clean, customizable Gradio 6 leaderboard template.",
+    version="2.0.0",
+)
+server.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
+
+
+@server.get("/", response_class=HTMLResponse, include_in_schema=False)
+async def homepage() -> HTMLResponse:
+    return HTMLResponse((STATIC_DIR / "index.html").read_text())
+
+
+@server.api(name="leaderboard_data", queue=False, concurrency_limit=None)
+def leaderboard_data() -> dict[str, Any]:
+    """Return the current leaderboard, evaluation queues, filters, and template copy."""
+    with DATA_LOCK:
+        return dict(DATA) if DATA else _empty_payload("Data is still loading.")
+
+
+@server.api(name="submit_model", concurrency_limit=4)
+def submit_model(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+) -> dict[str, str]:
+    """Validate a public Hugging Face model and add it to the evaluation queue."""
+    result = add_new_eval(model, base_model, revision, precision, weight_type, model_type)
+    if result["status"] == "success":
+        refresh_data(download=False)
+    return result
+
+
+if __name__ == "__main__":
+    refresh_data()
+    scheduler = BackgroundScheduler(daemon=True)
+    scheduler.add_job(refresh_data, "interval", minutes=10, max_instances=1, coalesce=True)
+    scheduler.start()
+    server.launch()

pyproject.toml

@@ -0,0 +1,14 @@
+[tool.ruff]
+line-length = 119
+
+[tool.ruff.lint]
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,9 @@
+APScheduler
+gradio==6.26.0
+huggingface-hub>=0.18.0
+numpy
+pandas
+python-dateutil
+sentencepiece
+tokenizers>=0.15.0
+transformers

src/about.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    task0 = Task("anli_r1", "acc", "ANLI")
+    task1 = Task("logiqa", "acc_norm", "LogiQA")
+
+
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
+
+
+# Your leaderboard name
+TITLE = "Open Leaderboard"
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = """
+## How it works
+
+## Reproducibility
+To reproduce our results, here is the commands you can run:
+
+"""
+
+EVALUATION_QUEUE_TEXT = """
+## Some good practices before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+"""

src/display/formatting.py

@@ -0,0 +1,15 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,114 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+from src.about import Tasks
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass(frozen=True)
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = ""  # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
+
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]

src/envs.py

@@ -0,0 +1,24 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
+
+OWNER = "demo-leaderboard-backend"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH = os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,195 @@
+import glob
+import json
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
+
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
+    model: str
+    revision: str  # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = ""  # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,
+            revision=config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture,
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(
+                f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}"
+            )
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if req_content["status"] in ["FINISHED"] and req_content["precision"] == precision.split(".")[-1]:
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, directories, files in os.walk(results_path):
+        directories[:] = [directory for directory in directories if not directory.startswith(".")]
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict()  # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -0,0 +1,62 @@
+import json
+import os
+
+import pandas as pd
+
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            entry_path = os.path.join(save_path, entry)
+            sub_entries = [
+                name
+                for name in os.listdir(entry_path)
+                if os.path.isfile(os.path.join(entry_path, name)) and not name.startswith(".")
+            ]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(entry_path, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -0,0 +1,107 @@
+import json
+import os
+from collections import defaultdict
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+
+def is_model_on_hub(
+    model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False
+) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(
+            model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
+        )
+        if test_tokenizer:
+            try:
+                AutoTokenizer.from_pretrained(
+                    model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
+                )
+            except ValueError as e:
+                return (False, f"uses a tokenizer which is not in a transformers release: {e}", None)
+            except Exception:
+                return (
+                    False,
+                    "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?",
+                    None,
+                )
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None,
+        )
+
+    except Exception:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,127 @@
+import json
+import os
+import re
+from datetime import datetime, timezone
+
+from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+MODEL_ID_PATTERN = re.compile(r"^[\w.-]+/[\w.-]+$")
+
+
+def response(status: str, message: str) -> dict[str, str]:
+    return {"status": status, "message": message}
+
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if REQUESTED_MODELS is None:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    model = model.strip()
+    base_model = base_model.strip()
+    revision = revision.strip() or "main"
+    if not MODEL_ID_PATTERN.fullmatch(model):
+        return response("error", "Enter a model ID in the form organization/model-name.")
+
+    user_name, model_path = model.split("/", 1)
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if not model_type:
+        return response("error", "Please select a model type.")
+
+    if weight_type in ["Delta", "Adapter"]:
+        if not MODEL_ID_PATTERN.fullmatch(base_model):
+            return response("error", "Enter a base model ID in the form organization/model-name.")
+        base_model_on_hub, error, _ = is_model_on_hub(
+            model_name=base_model,
+            revision=revision,
+            token=TOKEN,
+            test_tokenizer=True,
+        )
+        if not base_model_on_hub:
+            return response("error", f'Base model "{base_model}" {error}')
+
+    if weight_type != "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(
+            model_name=model,
+            revision=revision,
+            token=TOKEN,
+            test_tokenizer=True,
+        )
+        if not model_on_hub:
+            return response("error", f'Model "{model}" {error}')
+
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return response("error", "Could not read this model. Check its ID and revision, then try again.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+    try:
+        license_name = model_info.cardData["license"]
+    except Exception:
+        return response("error", "Please select a license in your model card.")
+
+    modelcard_ok, error_msg = check_model_card(model)
+    if not modelcard_ok:
+        return response("error", error_msg)
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license_name,
+        "private": False,
+    }
+
+    request_key = f"{model}_{revision}_{precision}"
+    if request_key in REQUESTED_MODELS:
+        return response("warning", "This model has already been submitted.")
+
+    out_dir = os.path.join(EVAL_REQUESTS_PATH, user_name)
+    os.makedirs(out_dir, exist_ok=True)
+    out_path = os.path.join(out_dir, f"{model_path}_eval_request_False_{precision}_{weight_type}.json")
+    with open(out_path, "w") as file:
+        json.dump(eval_entry, file)
+
+    try:
+        API.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=os.path.relpath(out_path, EVAL_REQUESTS_PATH),
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model} to eval queue",
+        )
+    except Exception:
+        os.remove(out_path)
+        return response("error", "The model passed validation, but the queue could not be updated. Please try again.")
+
+    REQUESTED_MODELS.add(request_key)
+    return response(
+        "success",
+        "Your model is in the evaluation queue. It should appear in the pending list shortly.",
+    )

static/app.js

@@ -0,0 +1,199 @@
+import { client } from "https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js";
+
+const state = {
+  api: null,
+  payload: null,
+  rows: [],
+  sort: { key: "Average ⬆️", direction: "desc" },
+};
+
+let columns = [];
+
+function configureColumns(benchmarks = []) {
+  columns = [
+    { key: "rank", label: "Rank" },
+    { key: "Model", label: "Model" },
+    { key: "Average ⬆️", label: "Average", score: true },
+    ...benchmarks.map((benchmark) => ({ key: benchmark, label: benchmark, score: true })),
+    { key: "Type", label: "Type" },
+    { key: "Precision", label: "Precision" },
+    { key: "#Params (B)", label: "Params (B)", score: true },
+    { key: "Hub License", label: "License" },
+    { key: "Hub ❤️", label: "Likes", score: true },
+    { key: "Available on the hub", label: "Hub" },
+  ];
+}
+
+const $ = (selector) => document.querySelector(selector);
+const $$ = (selector) => [...document.querySelectorAll(selector)];
+const escapeHtml = (value) => String(value ?? "").replace(/[&<>'"]/g, (char) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", "'": "&#39;", '"': "&quot;" })[char]);
+const safeModelId = (value) => /^[\w.-]+\/[\w.-]+$/.test(String(value || "")) ? value : "";
+const formatScore = (value) => value == null || Number.isNaN(Number(value)) ? "—" : Number(value).toFixed(2);
+const firstData = (response) => Array.isArray(response.data) ? response.data[0] : response.data;
+
+function markdown(source = "") {
+  let text = escapeHtml(source.trim());
+  text = text.replace(/```(?:[a-zA-Z0-9_+-]+)?\n([\s\S]*?)```/g, (_, code) => `<pre><code>${code.trim()}</code></pre>`);
+  text = text.replace(/^### (.+)$/gm, "<h3>$1</h3>").replace(/^## (.+)$/gm, "<h2>$1</h2>");
+  text = text.replace(/\[([^\]]+)\]\((https?:\/\/[^)]+)\)/g, '<a href="$2" target="_blank" rel="noreferrer">$1</a>');
+  text = text.replace(/`([^`]+)`/g, "<code>$1</code>");
+  return text.split(/\n{2,}/).map((block) => /^<(h2|h3|pre)/.test(block) ? block : `<p>${block.replace(/\n/g, "<br>")}</p>`).join("");
+}
+
+function renderHead() {
+  $("#leaderboard-head").innerHTML = `<tr>${columns.map((column) => {
+    const sortable = column.score || column.key === "Model";
+    const active = state.sort.key === column.key;
+    const arrow = active ? (state.sort.direction === "asc" ? " ↑" : " ↓") : "";
+    return `<th class="${sortable ? "sortable" : ""} ${active ? "active-sort" : ""}" data-key="${escapeHtml(column.key)}">${column.label}${arrow}</th>`;
+  }).join("")}</tr>`;
+  $$("th.sortable").forEach((header) => header.addEventListener("click", () => {
+    const key = header.dataset.key;
+    state.sort = { key, direction: state.sort.key === key && state.sort.direction === "desc" ? "asc" : "desc" };
+    renderHead();
+    renderRows();
+  }));
+}
+
+function filteredRows() {
+  const query = $("#search").value.trim().toLowerCase();
+  const type = $("#type-filter").value;
+  const precision = $("#precision-filter").value;
+  const available = $("#available-filter").checked;
+  const rows = state.rows.filter((row) => {
+    const haystack = `${row.Model || ""} ${row["Hub License"] || ""}`.toLowerCase();
+    return (!query || haystack.includes(query)) && (!type || row.Type === type) && (!precision || row.Precision === precision) && (!available || row["Available on the hub"] === true);
+  });
+  const multiplier = state.sort.direction === "asc" ? 1 : -1;
+  return rows.sort((a, b) => {
+    const av = a[state.sort.key];
+    const bv = b[state.sort.key];
+    if (typeof av === "number" && typeof bv === "number") return (av - bv) * multiplier;
+    return String(av ?? "").localeCompare(String(bv ?? "")) * multiplier;
+  });
+}
+
+function cell(row, column, rank) {
+  const value = row[column.key];
+  if (column.key === "rank") return `<td class="rank ${rank < 3 ? "top" : ""}">${String(rank + 1).padStart(2, "0")}</td>`;
+  if (column.key === "Model") {
+    const id = safeModelId(value);
+    return `<td class="model-cell">${id ? `<a href="https://huggingface.co/${encodeURI(id)}" target="_blank" rel="noreferrer">${escapeHtml(id)}</a>` : escapeHtml(value)}</td>`;
+  }
+  if (column.key === "Available on the hub") return `<td><span class="availability ${value === true ? "yes" : ""}" title="${value === true ? "Available" : "Unavailable"}"></span></td>`;
+  if (["Type", "Precision", "Hub License"].includes(column.key)) return `<td><span class="pill">${escapeHtml(value || "—")}</span></td>`;
+  if (column.score) return `<td class="score ${column.key === "Average ⬆️" ? "primary" : ""}">${formatScore(value)}</td>`;
+  return `<td>${escapeHtml(value ?? "—")}</td>`;
+}
+
+function renderRows() {
+  const rows = filteredRows();
+  $("#leaderboard-body").innerHTML = rows.map((row, rank) => `<tr>${columns.map((column) => cell(row, column, rank)).join("")}</tr>`).join("");
+  $(".table-shell").classList.toggle("is-empty", rows.length === 0);
+  $("#empty-state").classList.toggle("hidden", rows.length > 0);
+  $("#result-count").textContent = `${rows.length} model${rows.length === 1 ? "" : "s"} shown`;
+}
+
+function populateSelect(selector, values, selected = "") {
+  const select = $(selector);
+  const existing = select.querySelector("option")?.outerHTML || "";
+  select.innerHTML = existing + values.map((value) => `<option value="${escapeHtml(value)}" ${value === selected ? "selected" : ""}>${escapeHtml(value)}</option>`).join("");
+}
+
+function renderQueues(queues) {
+  const definitions = [
+    ["finished", "Finished", "Recently completed"],
+    ["running", "Running", "Evaluations in progress"],
+    ["pending", "Pending", "Waiting to be evaluated"],
+  ];
+  $("#queue-grid").innerHTML = definitions.map(([key, label, help]) => {
+    const rows = queues[key] || [];
+    const latest = rows[0]?.model ? `Latest: ${escapeHtml(rows[0].model)}` : help;
+    return `<article class="queue-card ${key}"><header><h3>${label}</h3><strong>${rows.length}</strong></header><p>${latest}</p></article>`;
+  }).join("");
+}
+
+function applyPayload(payload) {
+  state.payload = payload;
+  state.rows = payload.rows || [];
+  configureColumns(payload.benchmarks);
+  document.title = payload.copy?.title || "Open Leaderboard";
+  $("#leaderboard-name").textContent = payload.copy?.title || "Open Leaderboard";
+  $("#updated-at").textContent = `Updated ${new Intl.DateTimeFormat(undefined, { dateStyle: "medium", timeStyle: "short" }).format(new Date(payload.updated_at))}`;
+  $("#data-warning").textContent = payload.error || "";
+  $("#data-warning").classList.toggle("hidden", !payload.error);
+
+  const types = [...new Set(state.rows.map((row) => row.Type).filter(Boolean))];
+  const precisions = [...new Set(state.rows.map((row) => row.Precision).filter(Boolean))];
+  populateSelect("#type-filter", types);
+  populateSelect("#precision-filter", precisions);
+  populateSelect('[name="model_type"]', payload.model_types || []);
+  populateSelect('[name="precision"]', payload.precisions || [], "float16");
+  populateSelect('[name="weight_type"]', payload.weight_types || [], "Original");
+
+  renderHead();
+  renderRows();
+  renderQueues(payload.queues || {});
+  $("#submission-copy").innerHTML = markdown(payload.copy?.submission);
+  $("#about-copy").innerHTML = markdown(payload.copy?.about);
+  if (payload.copy?.citation) {
+    $("#citation-card").classList.remove("hidden");
+    $("#citation-label").textContent = payload.copy.citation_label;
+    $("#citation").textContent = payload.copy.citation;
+  }
+}
+
+async function loadData() {
+  try {
+    state.api = await client(location.origin);
+    const response = await state.api.predict("/leaderboard_data", {});
+    applyPayload(firstData(response));
+  } catch (error) {
+    $("#leaderboard-body").innerHTML = "";
+    $(".table-shell").classList.add("is-empty");
+    $("#data-warning").textContent = `The leaderboard could not be loaded: ${error.message}`;
+    $("#data-warning").classList.remove("hidden");
+    $("#empty-state").classList.remove("hidden");
+  }
+}
+
+$$('.tab').forEach((tab) => tab.addEventListener("click", () => {
+  $$('.tab').forEach((item) => item.classList.toggle("active", item === tab));
+  $$('.panel').forEach((panel) => panel.classList.toggle("active", panel.id === `panel-${tab.dataset.tab}`));
+  history.replaceState(null, "", `#${tab.dataset.tab}`);
+}));
+
+[$("#search"), $("#type-filter"), $("#precision-filter"), $("#available-filter")].forEach((control) => control.addEventListener("input", renderRows));
+
+$("#submit-form").addEventListener("submit", async (event) => {
+  event.preventDefault();
+  const button = event.currentTarget.querySelector("button[type=submit]");
+  const result = $("#submission-result");
+  const values = Object.fromEntries(new FormData(event.currentTarget));
+  button.disabled = true;
+  button.firstElementChild.textContent = "Validating model…";
+  result.className = "notice hidden";
+  try {
+    const response = await state.api.predict("/submit_model", values);
+    const data = firstData(response);
+    result.textContent = data.message;
+    result.className = `notice ${data.status}`;
+    if (data.status === "success") event.currentTarget.reset();
+  } catch (error) {
+    result.textContent = `Submission failed: ${error.message}`;
+    result.className = "notice error";
+  } finally {
+    button.disabled = false;
+    button.firstElementChild.textContent = "Submit evaluation";
+  }
+});
+
+$("#copy-citation").addEventListener("click", async () => {
+  await navigator.clipboard.writeText(state.payload.copy.citation);
+  $("#copy-citation").textContent = "Copied";
+  setTimeout(() => $("#copy-citation").textContent = "Copy citation", 1600);
+});
+
+const initialTab = location.hash.slice(1);
+if (["submit", "about"].includes(initialTab)) $(`.tab[data-tab="${initialTab}"]`).click();
+loadData();

static/index.html

@@ -0,0 +1,92 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="description" content="A customizable leaderboard for comparing model evaluation results." />
+    <title>Open Leaderboard</title>
+    <link rel="stylesheet" href="/static/styles.css" />
+  </head>
+  <body>
+    <main>
+      <header class="app-header">
+        <h1 id="leaderboard-name">Open Leaderboard</h1>
+      </header>
+
+      <nav class="tabs" aria-label="Leaderboard sections">
+        <button class="tab active" data-tab="leaderboard">
+          <svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 20V10M10 20V4M16 20v-7M22 20H2" /></svg>
+          <span>Leaderboard</span>
+        </button>
+        <button class="tab" data-tab="submit">
+          <svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 16V3M7 8l5-5 5 5M5 21h14" /></svg>
+          <span>Submit</span>
+        </button>
+        <button class="tab" data-tab="about">
+          <svg viewBox="0 0 24 24" aria-hidden="true"><circle cx="12" cy="12" r="9" /><path d="M12 11v5M12 8h.01" /></svg>
+          <span>About</span>
+        </button>
+      </nav>
+
+      <section class="panel active" id="panel-leaderboard">
+        <div class="panel-heading">
+          <div><h2>LLM Benchmark</h2><p>Compare evaluation results across models.</p></div>
+          <p id="updated-at" class="updated">Loading the latest results…</p>
+        </div>
+
+        <div class="toolbar">
+          <label class="search"><span>⌕</span><input id="search" type="search" placeholder="Search models or licenses" /></label>
+          <label class="select-control"><span>Model type</span><select id="type-filter"><option value="">All types</option></select></label>
+          <label class="select-control"><span>Precision</span><select id="precision-filter"><option value="">Any precision</option></select></label>
+          <label class="toggle"><input id="available-filter" type="checkbox" /> Available only</label>
+        </div>
+
+        <div id="data-warning" class="notice warning hidden"></div>
+        <div class="table-shell">
+          <table>
+            <thead id="leaderboard-head"></thead>
+            <tbody id="leaderboard-body">
+              <tr class="loading-row"><td><span class="skeleton"></span></td><td><span class="skeleton wide"></span></td><td><span class="skeleton"></span></td><td><span class="skeleton"></span></td></tr>
+            </tbody>
+          </table>
+          <div class="empty-state hidden" id="empty-state"><h3>No models found</h3><p>Try clearing one of your filters.</p></div>
+        </div>
+        <div class="table-footer"><span id="result-count">—</span><span>Click a score heading to sort</span></div>
+      </section>
+
+      <section class="panel" id="panel-submit">
+        <div class="panel-heading">
+          <div><h2>Submit a model</h2><p>Validate a public Hugging Face model and add it to the evaluation queue.</p></div>
+        </div>
+
+        <div class="queue-grid" id="queue-grid"></div>
+
+        <div class="submit-layout">
+          <form id="submit-form" class="block submit-card">
+            <div class="block-heading"><h3>Model details</h3><p>Required fields must be completed before submission.</p></div>
+            <div class="form-grid">
+              <label class="field full"><span>Model ID <b>required</b></span><input name="model" placeholder="organization/model-name" required /></label>
+              <label class="field"><span>Revision</span><input name="revision" placeholder="main" /></label>
+              <label class="field"><span>Model type <b>required</b></span><select name="model_type" required><option value="">Choose a type</option></select></label>
+              <label class="field"><span>Precision <b>required</b></span><select name="precision" required></select></label>
+              <label class="field"><span>Weights <b>required</b></span><select name="weight_type" required></select></label>
+              <label class="field full"><span>Base model <small>for adapter or delta weights</small></span><input name="base_model" placeholder="organization/base-model" /></label>
+            </div>
+            <button class="primary-button" type="submit"><span>Submit evaluation</span></button>
+            <div id="submission-result" class="notice hidden" role="status"></div>
+          </form>
+
+          <aside class="block guidance-card"><h3>Before you submit</h3><div id="submission-copy" class="prose"></div></aside>
+        </div>
+      </section>
+
+      <section class="panel" id="panel-about">
+        <div class="panel-heading"><div><h2>About</h2><p>Methodology and reproducibility information.</p></div></div>
+        <div id="about-copy" class="block prose about-copy"></div>
+        <div id="citation-card" class="block citation-card hidden"><div><h3 id="citation-label"></h3></div><button id="copy-citation">Copy</button><pre id="citation"></pre></div>
+      </section>
+    </main>
+
+    <script type="module" src="/static/app.js"></script>
+  </body>
+</html>

static/styles.css

@@ -0,0 +1,143 @@
+:root {
+  color-scheme: light;
+  --background: #ffffff;
+  --block: #ffffff;
+  --text: #111827;
+  --body-text: #374151;
+  --muted: #6b7280;
+  --border: #e5e7eb;
+  --border-strong: #d1d5db;
+  --primary: #f97316;
+  --primary-hover: #ea580c;
+  --primary-soft: #fff7ed;
+  --success: #15803d;
+  --warning: #a16207;
+  --danger: #b91c1c;
+}
+
+* { box-sizing: border-box; }
+body { margin: 0; min-height: 100vh; background: var(--background); color: var(--text); font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; font-size: 14px; }
+button, input, select { font: inherit; }
+button { cursor: pointer; }
+main { width: min(1180px, calc(100% - 32px)); margin: 0 auto; padding: 36px 0 52px; }
+
+.app-header { max-width: 760px; margin: 0 auto 28px; text-align: center; }
+.app-header h1 { margin: 0 0 10px; font-size: 32px; line-height: 1.2; font-weight: 700; }
+.tabs { display: flex; justify-content: center; gap: 4px; margin-bottom: 24px; padding-bottom: 10px; border-bottom: 1px solid var(--border); }
+.tab { display: inline-flex; align-items: center; justify-content: center; gap: 6px; padding: 8px 14px; border: 1px solid transparent; border-radius: 8px; background: transparent; color: var(--muted); font-weight: 500; }
+.tab svg { width: 16px; height: 16px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: round; stroke-linejoin: round; }
+.tab:hover { color: var(--text); background: #f3f4f6; }
+.tab.active { color: var(--text); background: var(--block); border-color: var(--border); box-shadow: 0 1px 2px rgba(0, 0, 0, .04); }
+
+.panel { display: none; }
+.panel.active { display: block; }
+.panel-heading { display: flex; align-items: end; justify-content: space-between; gap: 24px; margin-bottom: 14px; }
+.panel-heading h2 { margin: 0 0 4px; font-size: 20px; line-height: 1.3; }
+.panel-heading p { margin: 0; color: var(--muted); font-size: 13px; line-height: 1.45; }
+.updated { white-space: nowrap; font-size: 11px !important; }
+
+.block, .toolbar, .table-shell, .queue-card { background: var(--block); border: 1px solid var(--border); border-radius: 10px; }
+.toolbar { display: grid; grid-template-columns: minmax(240px, 1fr) auto auto auto; gap: 10px; align-items: end; padding: 12px; margin-bottom: 12px; }
+.search { display: flex; align-items: center; gap: 8px; height: 38px; padding: 0 11px; border: 1px solid var(--border-strong); border-radius: 7px; background: var(--block); }
+.search > span { color: var(--muted); font-size: 18px; }
+.search input { width: 100%; border: 0; outline: 0; background: transparent; color: var(--text); font-size: 13px; }
+.select-control { display: grid; gap: 4px; color: var(--muted); font-size: 11px; }
+.select-control select { height: 38px; min-width: 130px; padding: 0 28px 0 10px; border: 1px solid var(--border-strong); border-radius: 7px; outline: 0; background: var(--block); color: var(--text); font-size: 13px; }
+.toggle { display: flex; align-items: center; gap: 7px; height: 38px; padding: 0 9px; color: var(--body-text); font-size: 12px; white-space: nowrap; }
+.toggle input { width: 15px; height: 15px; accent-color: var(--primary); }
+
+.table-shell { position: relative; overflow-x: auto; }
+.table-shell.is-empty { min-height: 180px; }
+table { width: 100%; border-collapse: collapse; font-size: 12px; }
+th { height: 42px; padding: 0 12px; border-bottom: 1px solid var(--border); background: #f9fafb; color: var(--muted); text-align: right; font-size: 11px; font-weight: 600; white-space: nowrap; }
+th:first-child, th:nth-child(2), td:first-child, td:nth-child(2) { text-align: left; }
+th.sortable { cursor: pointer; user-select: none; }
+th.sortable:hover, th.active-sort { color: var(--text); }
+td { height: 54px; padding: 0 12px; border-bottom: 1px solid var(--border); text-align: right; white-space: nowrap; }
+tbody tr:last-child td { border-bottom: 0; }
+tbody tr:hover { background: #fafafa; }
+.rank { width: 42px; color: var(--muted); }
+.rank.top, .score.primary { color: var(--primary-hover); font-weight: 600; }
+.model-cell a { display: inline-flex; align-items: center; gap: 5px; max-width: 290px; overflow: hidden; color: var(--text); font-weight: 600; text-decoration: none; text-overflow: ellipsis; }
+.model-cell a::after { content: "↗"; color: var(--muted); font-size: 10px; }
+.model-cell a:hover { color: var(--primary-hover); }
+.pill { display: inline-flex; padding: 3px 7px; border-radius: 6px; background: #f3f4f6; color: var(--body-text); font-size: 11px; }
+.availability { display: inline-block; width: 8px; height: 8px; border-radius: 50%; background: var(--danger); }
+.availability.yes { background: var(--success); }
+.table-footer { display: flex; justify-content: space-between; padding: 8px 3px 0; color: var(--muted); font-size: 11px; }
+.empty-state { position: absolute; inset: 43px 0 0; display: grid; place-content: center; text-align: center; color: var(--muted); }
+.empty-state h3 { margin: 0 0 4px; color: var(--body-text); font-size: 14px; }
+.empty-state p { margin: 0; font-size: 12px; }
+.skeleton { display: block; width: 48px; height: 10px; border-radius: 4px; background: #e5e7eb; }
+.skeleton.wide { width: 160px; }
+
+.queue-grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px; margin-bottom: 14px; }
+.queue-card { padding: 14px; }
+.queue-card header { display: flex; align-items: center; justify-content: space-between; }
+.queue-card h3 { margin: 0; font-size: 13px; }
+.queue-card strong { font-size: 18px; }
+.queue-card p { margin: 7px 0 0; overflow: hidden; color: var(--muted); font-size: 11px; text-overflow: ellipsis; white-space: nowrap; }
+.queue-card.finished strong { color: var(--success); }
+.queue-card.running strong { color: var(--primary-hover); }
+.queue-card.pending strong { color: var(--warning); }
+
+.submit-layout { display: grid; grid-template-columns: minmax(0, 1.15fr) minmax(300px, .85fr); gap: 14px; align-items: start; }
+.submit-card, .guidance-card, .about-copy, .citation-card { padding: 18px; }
+.block-heading { padding-bottom: 14px; border-bottom: 1px solid var(--border); }
+.block-heading h3, .guidance-card > h3 { margin: 0 0 4px; font-size: 15px; }
+.block-heading p { margin: 0; color: var(--muted); font-size: 12px; }
+.form-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 14px 12px; padding: 16px 0; }
+.field { display: grid; gap: 6px; }
+.field.full { grid-column: 1 / -1; }
+.field > span { color: var(--body-text); font-size: 12px; font-weight: 500; }
+.field b { margin-left: 3px; color: var(--muted); font-size: 10px; font-weight: 400; }
+.field small { color: var(--muted); font-weight: 400; }
+.field input, .field select { width: 100%; height: 39px; padding: 0 10px; border: 1px solid var(--border-strong); border-radius: 7px; outline: 0; background: var(--block); color: var(--text); font-size: 13px; }
+.field input:focus, .field select:focus, .search:focus-within { border-color: var(--primary); box-shadow: 0 0 0 2px var(--primary-soft); }
+.primary-button { min-height: 38px; padding: 0 14px; border: 1px solid var(--primary-hover); border-radius: 7px; background: var(--primary); color: #fff; font-weight: 600; }
+.primary-button:hover { background: var(--primary-hover); }
+.primary-button:disabled { opacity: .6; cursor: wait; }
+.guidance-card { max-height: 560px; overflow: auto; }
+
+.prose { color: var(--body-text); font-size: 13px; line-height: 1.6; }
+.prose h2, .prose h3 { color: var(--text); }
+.prose h2 { margin: 20px 0 6px; font-size: 17px; }
+.prose h3 { margin: 16px 0 5px; font-size: 14px; }
+.prose p { margin: 7px 0; }
+.prose pre { margin: 10px 0; overflow-wrap: anywhere; white-space: pre-wrap; padding: 11px; border: 1px solid var(--border); border-radius: 7px; background: #f9fafb; font: 12px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; }
+.prose code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .92em; }
+.prose pre code { font-size: inherit; }
+.prose a { color: #2563eb; }
+.about-copy { min-height: 160px; }
+.about-copy h2:first-child { margin-top: 0; }
+.citation-card { display: grid; grid-template-columns: 1fr auto; gap: 12px; margin-top: 12px; }
+.citation-card h3 { margin: 5px 0 0; font-size: 14px; }
+.citation-card button { align-self: start; padding: 6px 10px; border: 1px solid var(--border-strong); border-radius: 6px; background: var(--block); color: var(--text); }
+.citation-card pre { grid-column: 1 / -1; overflow: auto; margin: 0; padding: 10px; border-radius: 6px; background: #f9fafb; font: 12px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; }
+
+.notice { margin: 10px 0; padding: 10px 12px; border: 1px solid; border-radius: 7px; font-size: 12px; line-height: 1.45; }
+.notice.warning { color: var(--warning); background: #fffbeb; border-color: #fde68a; }
+.notice.error { color: var(--danger); background: #fef2f2; border-color: #fecaca; }
+.notice.success { color: var(--success); background: #f0fdf4; border-color: #bbf7d0; }
+.hidden { display: none !important; }
+@media (max-width: 820px) {
+  .toolbar { grid-template-columns: 1fr 1fr; }
+  .search { grid-column: 1 / -1; }
+  .submit-layout { grid-template-columns: 1fr; }
+  .guidance-card { max-height: none; }
+}
+
+@media (max-width: 560px) {
+  main { width: min(100% - 24px, 1180px); padding-top: 24px; }
+  .app-header h1 { font-size: 26px; }
+  .tabs { justify-content: stretch; }
+  .tab { flex: 1; padding-inline: 8px; }
+  .panel-heading { display: block; }
+  .updated { margin-top: 6px !important; }
+  .toolbar { grid-template-columns: 1fr; }
+  .search { grid-column: auto; }
+  .select-control select { width: 100%; }
+  .queue-grid, .form-grid { grid-template-columns: 1fr; }
+  .field.full { grid-column: auto; }
+  .table-footer span:last-child { display: none; }
+}