Add CREF

CLAUDE.md

@@ -30,7 +30,9 @@ No test or lint frameworks are configured.
 ```
 fev-leaderboard-app.py     # Main entry point (Streamlit multi-page router)
 save_tables.py             # Generates tables/ and src/vendor/ from a single fev commit (run at Docker build)
-pages/
+benchmarks.py              # Combined navigation of fev-bench and CREF, and the switcher between them
+fev_pages/                 # Not pages/: Streamlit would read that as a page folder and show
+                           # "Page not found" on the first visit to /cref after a restart
 ├── fev_bench.py           # Main leaderboard: computes leaderboard/pairwise, renders them
 └── about.py               # Help page with links
 src/
@@ -38,6 +40,7 @@ src/
 ├── strings.py             # UI text, metric descriptions, paper citations
 ├── task_groups.py         # Task groupings by frequency and domain
 └── vendor/                # GENERATED, gitignored: compute logic copied from the fev repo
+cref/                      # The CREF leaderboard (/cref pages); see cref/README.md
     └── fev_bench_compute.py
 tables/                    # GENERATED by save_tables.py, gitignored
 ├── summaries.csv          # Raw evaluation summaries (leaderboards are computed from these)

benchmarks.py

@@ -0,0 +1,55 @@
+import streamlit as st
+
+from cref import navigation
+
+SWITCHER_STYLE = """<style>
+.benchmarks {position: relative; width: fit-content; font-weight: 700}
+.benchmarks summary {display: flex; align-items: center; gap: 0.6rem; padding: 0.1rem 0.6rem 0.1rem 0.3rem;
+  margin-left: -0.3rem; border-radius: 0.5rem; font-size: 1.4rem; color: #31333f; cursor: pointer; list-style: none}
+.benchmarks summary::-webkit-details-marker {display: none}
+.benchmarks summary::after {content: ""; width: 0.45rem; height: 0.45rem; border: solid #6093c4;
+  border-width: 0 2.5px 2.5px 0; transform: translateY(-25%) rotate(45deg)}
+.benchmarks[open] summary::after {transform: translateY(25%) rotate(-135deg)}
+.benchmarks summary:hover, .benchmarks[open] summary {background: #f0f2f6}
+.benchmarks div {position: absolute; z-index: 1000; top: calc(100% + 0.3rem); left: -0.3rem; min-width: 10rem;
+  padding: 0.3rem; background: #fff; border: 1px solid #e3e6ea; border-radius: 0.6rem;
+  box-shadow: 0 6px 20px rgb(0 0 0 / 8%)}
+.benchmarks a {display: block; padding: 0.4rem 0.6rem; border-radius: 0.4rem; font-size: 1rem; color: #31333f;
+  text-decoration: none}
+.benchmarks a:hover {background: #f0f2f6}
+.benchmarks a[aria-current="page"] {color: #6093c4}
+</style>"""
+
+BENCHMARKS = [("fev-bench", "/"), ("CREF", "/cref")]
+
+
+# Plain links, so a switch loads the other benchmark afresh instead of keeping fev's sidebar.
+def switcher(current: str) -> str:
+    links = "".join(
+        f'<a href="{href}" aria-current="{"page" if name == current else "false"}">{name}</a>'
+        for name, href in BENCHMARKS
+    )
+    return f'{SWITCHER_STYLE}<details class="benchmarks"><summary>{current}</summary><div>{links}</div></details>'
+
+
+def run(fev: list[st.Page]) -> None:
+    cref = navigation.pages()
+    page = st.navigation(fev + cref, position="hidden")
+    # huggingface.co/spaces/autogluon/fev-bench?benchmark=cref: Hugging Face passes only the
+    # query string into the app, so this opens CREF on the first load.
+    if not st.session_state.get("routed"):
+        st.session_state["routed"] = True
+        if st.query_params.get("benchmark") == "cref" and page not in cref:
+            st.switch_page(cref[0])
+    if page in cref:
+        navigation.page_config()
+        with st.container(horizontal=True, vertical_alignment="center", gap="large"):
+            st.html(switcher("CREF"), width="content")
+            for link in cref:
+                st.page_link(link)
+    else:
+        with st.sidebar:
+            st.html(switcher("fev-bench"))
+            for link in fev:
+                st.page_link(link)
+    page.run()

cref/README.md

@@ -0,0 +1,5 @@
+# CREF leaderboard
+
+The CREF leaderboard, served at `/cref`. It reads the published leaderboards from
+`CREF_DATA_URL`, or from `cref/data/` when that is unset. Model names, types and links are
+in `models.toml`; the tasks are in `tasks.csv`.

cref/charts.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+import altair as alt
+import pandas as pd
+
+from cref import views
+from cref.views import COLUMNS
+
+PALETTE = [
+    "#4E79A7", "#F28E2B", "#59A14F", "#E15759", "#B07AA1", "#76B7B2", "#EDC948", "#9C755F",
+    "#FF9DA7", "#BAB0AC", "#A0CBE8", "#FFBE7D", "#8CD17D", "#FF9D9A", "#D4A6C8", "#86BCB6",
+    "#F1CE63", "#D7B5A6", "#499894", "#79706E", "#1F3A93", "#6B4226", "#2E7D32", "#8E24AA",
+]  # fmt: skip
+LEGEND_COLUMNS = 3
+PAIRWISE = {
+    "win_rate": ("Win rate (%)", [0, 100], 50),
+    "skill_score": ("Skill score (%)", [-40, 40], 0),
+}
+
+
+def skill_chart(df: pd.DataFrame, models: views.Models) -> alt.LayerChart:
+    data = views.competitors(df.dropna(subset=["skill_score"]), models)
+    labels = views.unique_labels(zip(data["model_id"], data["model"], strict=True))
+    data = pd.DataFrame(
+        {
+            "label": data["model_id"].map(labels),
+            "skill_score": data["skill_score"],
+            "type": [views.kind(m, models).label for m in data["model_id"]],
+        }
+    )
+    kinds = [k for k in [*views.MODEL_TYPES.values(), views.OTHER] if k.label in set(data["type"])]
+    y = alt.Y("label:N", sort=None, title=None, axis=alt.Axis(labelLimit=320, ticks=False))
+    x = alt.X("skill_score:Q", title=COLUMNS["skill_score"], axis=alt.Axis(grid=False))
+    bars = (
+        alt.Chart(data)
+        .mark_bar(cornerRadiusEnd=3)
+        .encode(
+            x=x,
+            y=y,
+            color=alt.Color(
+                "type:N",
+                scale=alt.Scale(domain=[k.label for k in kinds], range=[k.bar for k in kinds]),
+                title=None,
+                legend=alt.Legend(orient="top"),
+            ),
+            tooltip=[alt.Tooltip("skill_score:Q", title=COLUMNS["skill_score"], format=".1f")],
+        )
+    )
+    values = [
+        alt.Chart(data)
+        .transform_filter(test)
+        .mark_text(align=align, dx=dx, fontSize=11, color="#414D5C")
+        .encode(x=x, y=y, text=alt.Text("skill_score:Q", format=".1f"))
+        for test, align, dx in [
+            ("datum.skill_score >= 0", "left", 4),
+            ("datum.skill_score < 0", "right", -4),
+        ]
+    ]
+    return alt.layer(bars, *values).properties(height=max(120, 26 * len(data)))
+
+
+def pairwise_frame(
+    df: pd.DataFrame, models: views.Models, pairwise: dict[str, Any], value: str
+) -> tuple[pd.DataFrame, list[str]]:
+    shown = views.competitors(df, models)
+    labels = views.unique_labels(zip(shown["model_id"], shown["model"], strict=True))
+    cells = [
+        {"model": labels[a], "opponent": labels[b], "value": 100 * cell[value]}
+        for a in labels
+        for b, cell in pairwise.get(a, {}).items()
+        if b in labels and cell[value] is not None
+    ]
+    return pd.DataFrame(cells, columns=["model", "opponent", "value"]), list(labels.values())
+
+
+def pairwise_chart(cells: pd.DataFrame, order: Sequence[str], value: str) -> alt.LayerChart:
+    title, domain, mid = PAIRWISE[value]
+    far = (domain[1] - mid) * 0.6
+    axis = {"labelLimit": 320, "ticks": False, "domain": False}
+    base = alt.Chart(cells).encode(
+        x=alt.X(
+            "opponent:N",
+            sort=list(order),
+            title=None,
+            axis=alt.Axis(orient="top", labelAngle=-90, **axis),
+        ),
+        y=alt.Y("model:N", sort=list(order), title=None, axis=alt.Axis(**axis)),
+    )
+    heat = base.mark_rect().encode(
+        color=alt.Color(
+            "value:Q",
+            legend=None,
+            scale=alt.Scale(scheme="purplegreen", domain=domain, domainMid=mid, clamp=True),
+        ),
+        tooltip=[alt.Tooltip("value:Q", title=title, format=".1f")],
+    )
+    text = base.mark_text(fontSize=9).encode(
+        text=alt.Text("value:Q", format=".0f"),
+        color=alt.condition(
+            f"abs(datum.value - {mid}) > {far}", alt.value("#FFFFFF"), alt.value("#31333F")
+        ),
+    )
+    return alt.layer(heat, text).properties(height=alt.Step(22))
+
+
+def trend_frame(rows: list[dict[str, Any]], column: str, labels: dict[str, str]) -> pd.DataFrame:
+    df = pd.DataFrame(rows, columns=["date", "model_id", column])
+    df = df[df["model_id"].isin(list(labels))].dropna(subset=[column])
+    return pd.DataFrame(
+        {
+            "date": pd.to_datetime(df["date"]),
+            "value": 100 * df[column],
+            "model": df["model_id"].map(labels),
+        }
+    )
+
+
+def trend_chart(df: pd.DataFrame, order: Sequence[str], title: str) -> alt.Chart:
+    order = list(order)
+    picked = alt.selection_point(fields=["model"], bind="legend")
+    colors = [PALETTE[i % len(PALETTE)] for i in range(len(order))]
+    return (
+        alt.Chart(df)
+        .mark_line(point=True)
+        .encode(
+            x=alt.X("date:T", title=None, axis=alt.Axis(format="%b %d")),
+            y=alt.Y("value:Q", title=title, scale=alt.Scale(zero=False)),
+            color=alt.Color(
+                "model:N",
+                scale=alt.Scale(domain=order, range=colors),
+                title=None,
+                legend=alt.Legend(
+                    orient="bottom", columns=LEGEND_COLUMNS, symbolLimit=0, labelLimit=320
+                ),
+            ),
+            opacity=alt.condition(picked, alt.value(1.0), alt.value(0.12)),
+            tooltip=[
+                alt.Tooltip("date:T", title="Day"),
+                alt.Tooltip("value:Q", title=title, format=".1f"),
+            ],
+        )
+        .add_params(picked)
+        # The height includes the legend, about 30 px a row.
+        .properties(height=460 + 30 * -(-len(order) // LEGEND_COLUMNS))
+    )

cref/controls.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import streamlit as st
+
+from cref import source
+from cref import views
+
+
+@st.cache_resource(on_release=source.Source.close)
+def data_source(variant: str = "release") -> source.Source:
+    return source.Source.from_env(variant)
+
+
+@st.cache_resource(on_release=source.History.close)
+def history(variant: str = "release") -> source.History:
+    return source.History.from_env(variant)
+
+
+@st.cache_resource
+def models() -> views.Models:
+    return views.load_models()
+
+
+@st.cache_resource
+def tasks():
+    return views.load_tasks()
+
+
+def metric_control() -> str:
+    return st.segmented_control(
+        "Metric",
+        list(views.METRICS),
+        default="mase",
+        required=True,
+        format_func=views.METRICS.get,
+        help=views.METRIC_HELP,
+    )
+
+
+def type_controls(models: views.Models) -> set[str]:
+    present = [t for t in views.MODEL_TYPES if any(i.get("type") == t for i in models.values())]
+    if not present:
+        return set(views.MODEL_TYPES)
+    picked = st.pills(
+        "Model types",
+        present,
+        selection_mode="multi",
+        default=present,
+        format_func=lambda t: views.MODEL_TYPES[t].label,
+        help="Hides the other models. The seasonal naive baseline is always shown.",
+    )
+    return set(picked or ())
+
+
+def board_controls(models: views.Models) -> tuple[str, str, set[str]]:
+    variants = data_source("all").get().board is not None
+    cells = st.columns([1, 1.5, 3] if variants else [1, 4], vertical_alignment="bottom")
+    with cells[0]:
+        metric = metric_control()
+    variant = "release"
+    if variants:
+        with cells[1]:
+            variant = st.segmented_control(
+                "Pretrained samples counted",
+                list(views.VARIANTS),
+                default="release",
+                required=True,
+                format_func=views.VARIANTS.get,
+                help=views.VARIANT_HELP,
+            )
+    with cells[-1]:
+        types = type_controls(models)
+    return metric, variant, types