fev-bench / cref /charts.py
AndiA's picture
Add CREF (#4)
585789a
Raw History Blame Contribute Delete
5.61 kB
from __future__ import annotations
from collections.abc import Sequence
from typing import Any
import altair as alt
import pandas as pd
from cref import views
from cref.views import COLUMNS
PALETTE = [
"#4E79A7", "#F28E2B", "#59A14F", "#E15759", "#B07AA1", "#76B7B2", "#EDC948", "#9C755F",
"#FF9DA7", "#BAB0AC", "#A0CBE8", "#FFBE7D", "#8CD17D", "#FF9D9A", "#D4A6C8", "#86BCB6",
"#F1CE63", "#D7B5A6", "#499894", "#79706E", "#1F3A93", "#6B4226", "#2E7D32", "#8E24AA",
] # fmt: skip
LEGEND_COLUMNS = 3
PAIRWISE = {
"win_rate": ("Win rate (%)", [0, 100], 50),
"skill_score": ("Skill score (%)", [-40, 40], 0),
}
def skill_chart(df: pd.DataFrame, models: views.Models) -> alt.LayerChart:
data = views.competitors(df.dropna(subset=["skill_score"]), models)
labels = views.unique_labels(zip(data["model_id"], data["model"], strict=True))
data = pd.DataFrame(
{
"label": data["model_id"].map(labels),
"skill_score": data["skill_score"],
"type": [views.kind(m, models).label for m in data["model_id"]],
}
)
kinds = [k for k in [*views.MODEL_TYPES.values(), views.OTHER] if k.label in set(data["type"])]
y = alt.Y("label:N", sort=None, title=None, axis=alt.Axis(labelLimit=320, ticks=False))
x = alt.X("skill_score:Q", title=COLUMNS["skill_score"], axis=alt.Axis(grid=False))
bars = (
alt.Chart(data)
.mark_bar(cornerRadiusEnd=3)
.encode(
x=x,
y=y,
color=alt.Color(
"type:N",
scale=alt.Scale(domain=[k.label for k in kinds], range=[k.bar for k in kinds]),
title=None,
legend=alt.Legend(orient="top"),
),
tooltip=[alt.Tooltip("skill_score:Q", title=COLUMNS["skill_score"], format=".1f")],
)
)
values = [
alt.Chart(data)
.transform_filter(test)
.mark_text(align=align, dx=dx, fontSize=11, color="#414D5C")
.encode(x=x, y=y, text=alt.Text("skill_score:Q", format=".1f"))
for test, align, dx in [
("datum.skill_score >= 0", "left", 4),
("datum.skill_score < 0", "right", -4),
]
]
return alt.layer(bars, *values).properties(height=max(120, 26 * len(data)))
def pairwise_frame(
df: pd.DataFrame, models: views.Models, pairwise: dict[str, Any], value: str
) -> tuple[pd.DataFrame, list[str]]:
shown = views.competitors(df, models)
labels = views.unique_labels(zip(shown["model_id"], shown["model"], strict=True))
cells = [
{"model": labels[a], "opponent": labels[b], "value": 100 * cell[value]}
for a in labels
for b, cell in pairwise.get(a, {}).items()
if b in labels and cell[value] is not None
]
return pd.DataFrame(cells, columns=["model", "opponent", "value"]), list(labels.values())
def pairwise_chart(cells: pd.DataFrame, order: Sequence[str], value: str) -> alt.LayerChart:
title, domain, mid = PAIRWISE[value]
far = (domain[1] - mid) * 0.6
axis = {"labelLimit": 320, "ticks": False, "domain": False}
base = alt.Chart(cells).encode(
x=alt.X(
"opponent:N",
sort=list(order),
title=None,
axis=alt.Axis(orient="top", labelAngle=-90, **axis),
),
y=alt.Y("model:N", sort=list(order), title=None, axis=alt.Axis(**axis)),
)
heat = base.mark_rect().encode(
color=alt.Color(
"value:Q",
legend=None,
scale=alt.Scale(scheme="purplegreen", domain=domain, domainMid=mid, clamp=True),
),
tooltip=[alt.Tooltip("value:Q", title=title, format=".1f")],
)
text = base.mark_text(fontSize=9).encode(
text=alt.Text("value:Q", format=".0f"),
color=alt.condition(
f"abs(datum.value - {mid}) > {far}", alt.value("#FFFFFF"), alt.value("#31333F")
),
)
return alt.layer(heat, text).properties(height=alt.Step(22))
def trend_frame(rows: list[dict[str, Any]], column: str, labels: dict[str, str]) -> pd.DataFrame:
df = pd.DataFrame(rows, columns=["date", "model_id", column])
df = df[df["model_id"].isin(list(labels))].dropna(subset=[column])
return pd.DataFrame(
{
"date": pd.to_datetime(df["date"]),
"value": 100 * df[column],
"model": df["model_id"].map(labels),
}
)
def trend_chart(df: pd.DataFrame, order: Sequence[str], title: str) -> alt.Chart:
order = list(order)
picked = alt.selection_point(fields=["model"], bind="legend")
colors = [PALETTE[i % len(PALETTE)] for i in range(len(order))]
return (
alt.Chart(df)
.mark_line(point=True)
.encode(
x=alt.X("date:T", title=None, axis=alt.Axis(format="%b %d")),
y=alt.Y("value:Q", title=title, scale=alt.Scale(zero=False)),
color=alt.Color(
"model:N",
scale=alt.Scale(domain=order, range=colors),
title=None,
legend=alt.Legend(
orient="bottom", columns=LEGEND_COLUMNS, symbolLimit=0, labelLimit=320
),
),
opacity=alt.condition(picked, alt.value(1.0), alt.value(0.12)),
tooltip=[
alt.Tooltip("date:T", title="Day"),
alt.Tooltip("value:Q", title=title, format=".1f"),
],
)
.add_params(picked)
# The height includes the legend, about 30 px a row.
.properties(height=460 + 30 * -(-len(order) // LEGEND_COLUMNS))
)