Spaces:
Running
Running
Download cref/charts.py from autogluon/fev-bench: direct link, hf CLI and curl.
- Browser
- Download file 5.61 kB
-
https://huggingface.co/spaces/autogluon/fev-bench/resolve/main/cref/charts.py
- Command line
-
hf download hf://spaces/autogluon/fev-bench/cref/charts.py
-
curl -L -o charts.py https://huggingface.co/spaces/autogluon/fev-bench/resolve/main/cref/charts.py
5.61 kB
| from __future__ import annotations | |
| from collections.abc import Sequence | |
| from typing import Any | |
| import altair as alt | |
| import pandas as pd | |
| from cref import views | |
| from cref.views import COLUMNS | |
| PALETTE = [ | |
| "#4E79A7", "#F28E2B", "#59A14F", "#E15759", "#B07AA1", "#76B7B2", "#EDC948", "#9C755F", | |
| "#FF9DA7", "#BAB0AC", "#A0CBE8", "#FFBE7D", "#8CD17D", "#FF9D9A", "#D4A6C8", "#86BCB6", | |
| "#F1CE63", "#D7B5A6", "#499894", "#79706E", "#1F3A93", "#6B4226", "#2E7D32", "#8E24AA", | |
| ] # fmt: skip | |
| LEGEND_COLUMNS = 3 | |
| PAIRWISE = { | |
| "win_rate": ("Win rate (%)", [0, 100], 50), | |
| "skill_score": ("Skill score (%)", [-40, 40], 0), | |
| } | |
| def skill_chart(df: pd.DataFrame, models: views.Models) -> alt.LayerChart: | |
| data = views.competitors(df.dropna(subset=["skill_score"]), models) | |
| labels = views.unique_labels(zip(data["model_id"], data["model"], strict=True)) | |
| data = pd.DataFrame( | |
| { | |
| "label": data["model_id"].map(labels), | |
| "skill_score": data["skill_score"], | |
| "type": [views.kind(m, models).label for m in data["model_id"]], | |
| } | |
| ) | |
| kinds = [k for k in [*views.MODEL_TYPES.values(), views.OTHER] if k.label in set(data["type"])] | |
| y = alt.Y("label:N", sort=None, title=None, axis=alt.Axis(labelLimit=320, ticks=False)) | |
| x = alt.X("skill_score:Q", title=COLUMNS["skill_score"], axis=alt.Axis(grid=False)) | |
| bars = ( | |
| alt.Chart(data) | |
| .mark_bar(cornerRadiusEnd=3) | |
| .encode( | |
| x=x, | |
| y=y, | |
| color=alt.Color( | |
| "type:N", | |
| scale=alt.Scale(domain=[k.label for k in kinds], range=[k.bar for k in kinds]), | |
| title=None, | |
| legend=alt.Legend(orient="top"), | |
| ), | |
| tooltip=[alt.Tooltip("skill_score:Q", title=COLUMNS["skill_score"], format=".1f")], | |
| ) | |
| ) | |
| values = [ | |
| alt.Chart(data) | |
| .transform_filter(test) | |
| .mark_text(align=align, dx=dx, fontSize=11, color="#414D5C") | |
| .encode(x=x, y=y, text=alt.Text("skill_score:Q", format=".1f")) | |
| for test, align, dx in [ | |
| ("datum.skill_score >= 0", "left", 4), | |
| ("datum.skill_score < 0", "right", -4), | |
| ] | |
| ] | |
| return alt.layer(bars, *values).properties(height=max(120, 26 * len(data))) | |
| def pairwise_frame( | |
| df: pd.DataFrame, models: views.Models, pairwise: dict[str, Any], value: str | |
| ) -> tuple[pd.DataFrame, list[str]]: | |
| shown = views.competitors(df, models) | |
| labels = views.unique_labels(zip(shown["model_id"], shown["model"], strict=True)) | |
| cells = [ | |
| {"model": labels[a], "opponent": labels[b], "value": 100 * cell[value]} | |
| for a in labels | |
| for b, cell in pairwise.get(a, {}).items() | |
| if b in labels and cell[value] is not None | |
| ] | |
| return pd.DataFrame(cells, columns=["model", "opponent", "value"]), list(labels.values()) | |
| def pairwise_chart(cells: pd.DataFrame, order: Sequence[str], value: str) -> alt.LayerChart: | |
| title, domain, mid = PAIRWISE[value] | |
| far = (domain[1] - mid) * 0.6 | |
| axis = {"labelLimit": 320, "ticks": False, "domain": False} | |
| base = alt.Chart(cells).encode( | |
| x=alt.X( | |
| "opponent:N", | |
| sort=list(order), | |
| title=None, | |
| axis=alt.Axis(orient="top", labelAngle=-90, **axis), | |
| ), | |
| y=alt.Y("model:N", sort=list(order), title=None, axis=alt.Axis(**axis)), | |
| ) | |
| heat = base.mark_rect().encode( | |
| color=alt.Color( | |
| "value:Q", | |
| legend=None, | |
| scale=alt.Scale(scheme="purplegreen", domain=domain, domainMid=mid, clamp=True), | |
| ), | |
| tooltip=[alt.Tooltip("value:Q", title=title, format=".1f")], | |
| ) | |
| text = base.mark_text(fontSize=9).encode( | |
| text=alt.Text("value:Q", format=".0f"), | |
| color=alt.condition( | |
| f"abs(datum.value - {mid}) > {far}", alt.value("#FFFFFF"), alt.value("#31333F") | |
| ), | |
| ) | |
| return alt.layer(heat, text).properties(height=alt.Step(22)) | |
| def trend_frame(rows: list[dict[str, Any]], column: str, labels: dict[str, str]) -> pd.DataFrame: | |
| df = pd.DataFrame(rows, columns=["date", "model_id", column]) | |
| df = df[df["model_id"].isin(list(labels))].dropna(subset=[column]) | |
| return pd.DataFrame( | |
| { | |
| "date": pd.to_datetime(df["date"]), | |
| "value": 100 * df[column], | |
| "model": df["model_id"].map(labels), | |
| } | |
| ) | |
| def trend_chart(df: pd.DataFrame, order: Sequence[str], title: str) -> alt.Chart: | |
| order = list(order) | |
| picked = alt.selection_point(fields=["model"], bind="legend") | |
| colors = [PALETTE[i % len(PALETTE)] for i in range(len(order))] | |
| return ( | |
| alt.Chart(df) | |
| .mark_line(point=True) | |
| .encode( | |
| x=alt.X("date:T", title=None, axis=alt.Axis(format="%b %d")), | |
| y=alt.Y("value:Q", title=title, scale=alt.Scale(zero=False)), | |
| color=alt.Color( | |
| "model:N", | |
| scale=alt.Scale(domain=order, range=colors), | |
| title=None, | |
| legend=alt.Legend( | |
| orient="bottom", columns=LEGEND_COLUMNS, symbolLimit=0, labelLimit=320 | |
| ), | |
| ), | |
| opacity=alt.condition(picked, alt.value(1.0), alt.value(0.12)), | |
| tooltip=[ | |
| alt.Tooltip("date:T", title="Day"), | |
| alt.Tooltip("value:Q", title=title, format=".1f"), | |
| ], | |
| ) | |
| .add_params(picked) | |
| # The height includes the legend, about 30 px a row. | |
| .properties(height=460 + 30 * -(-len(order) // LEGEND_COLUMNS)) | |
| ) | |