fev-bench / cref /views.py
AndiA's picture
Fetch from live bucket (#6)
b2c1aa1
Raw History Blame Contribute Delete
11.8 kB
from __future__ import annotations
import re
import tomllib
from collections import Counter
from collections.abc import Iterable
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, NamedTuple
from urllib.parse import quote, urlsplit
import pandas as pd
from pandas.io.formats.style import Styler
BASELINE = "__baseline__"
BASELINE_NAME = "Seasonal naive"
BREAKDOWNS = {"domain": "By domain", "freq": "By frequency", "task": "By task"}
FREQUENCIES = ("upto30min", "hourly", "daily", "weekly", "monthly")
GROUP_LABELS = {
"gov_services": "Gov. Services",
"public_interest": "Public Interest",
"upto30min": "≤ 30 min",
}
EMPTY_BOARD: dict[str, Any] = {"sample_count": 0, "tasks": {}, "rankings": []}
METRICS = {"mase": "MASE", "sql": "SQL"}
VARIANTS = {"release": "Live", "all": "Every sample"}
VARIANT_HELP = (
"A model's training data may leak a sample's actual values, directly or indirectly. Two "
"modes allow both a strict and a faster comparison of models. Where a forecast doesn't "
"count, the model gets the seasonal naive baseline's score.\n\n"
"**Live:** only what a model could have forecast live. A pretrained model counts "
"from its checkpoint's release, LLM-based and agentic models from their first live day.\n\n"
"**Every sample:** pretrained models count on every sample (unless "
"direct leakage is reported for specific samples; those fall back to the baseline), "
"LLM-based models from their training-data cutoff, agentic models from their first live day."
)
METRIC_HELP = (
"MASE: Mean Absolute Scaled Error (point forecast) "
"SQL: Scaled Quantile Loss (probabilistic forecast)"
)
class Kind(NamedTuple):
label: str
tag: str
bar: str
MODEL_TYPES = {
"statistical": Kind("Statistical", "#ECE6DD", "#A08C6E"),
"pretrained": Kind("Pretrained", "#E9DDF5", "#8D5EB7"),
"llm-based": Kind("LLM-based", "#DCE6F2", "#4E79A7"),
"agentic": Kind("Agentic", "#DCEEE4", "#3E9C6E"),
"closed-api": Kind("Closed API", "#F3DEDE", "#C44E52"),
}
OTHER = Kind("Other", "#ECEDEF", "#9AA0A6")
TABLE_ROWS = 20
ROW_HEIGHT = 35
MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"}
BASELINE_COLOR = "#7D8998"
RANK_COLORS = {1: "#F7D36B", 2: "#E5E7EB", 3: "#E6B089"}
COLUMNS = {
"link": "Model",
"skill_score": "Skill score (%)",
"win_rate": "Win rate (%)",
"marginal_win_rate": "Marginal win rate (%)",
"coverage_samples": "Coverage, samples (%)",
"coverage_weight": "Coverage, weight (%)",
"type": "Type",
}
HIDDEN_COLUMNS = ("model", "model_id")
TREND_VALUES = ("skill_score", "win_rate", "coverage_samples", "coverage_weight")
# The Model column links each name; its text is the URL's fragment.
LINK_TEXT = r"#(.*)$"
# The link column decodes its text after the "#" and reads "+" as a space.
def with_text(url: str, text: str) -> str:
return f"{url}#{quote(text, safe='')}"
ABOUT_PAGE = "cref-about"
MODELS = Path(__file__).parent / "models.toml"
TASKS = Path(__file__).parent / "tasks.csv"
TASK_COLUMNS = {
"name": "Task",
"domain": "Domain",
"frequency": "Frequency",
"horizon": "Horizon (steps)",
"source": "Source",
"license": "License",
}
MODEL_FIELDS = {"name", "url", "type", "baseline"}
MODEL_ID = re.compile(rf"[a-z0-9](?:[a-z0-9-]{{0,62}}[a-z0-9])?|{BASELINE}")
Models = dict[str, dict[str, Any]]
def is_https(url: object) -> bool:
if not isinstance(url, str) or any(c.isspace() or not c.isprintable() for c in url):
return False
parts = urlsplit(url)
return parts.scheme == "https" and bool(parts.netloc)
def load_models(path: Path = MODELS) -> Models:
models = tomllib.loads(path.read_text(encoding="utf-8"))
for model_id, info in models.items():
where = f"{path.name}: [{model_id}]"
if not MODEL_ID.fullmatch(model_id) or not isinstance(info, dict):
raise ValueError(f"{where}: not a model ID")
if unknown := set(info) - MODEL_FIELDS:
raise ValueError(f"{where}: unknown fields {sorted(unknown)}")
if "url" in info and (not is_https(info["url"]) or "#" in info["url"]):
raise ValueError(f"{where}: url is not an https:// link without #")
if "baseline" in info and not isinstance(info["baseline"], bool):
raise ValueError(f"{where}: baseline must be true or false")
name = info.get("name", "x")
if not isinstance(name, str) or not 0 < len(name) <= 256 or not name.isprintable():
raise ValueError(f"{where}: name must be 1 to 256 printable characters")
if "type" in info and info["type"] not in MODEL_TYPES:
raise ValueError(f"{where}: type must be one of {list(MODEL_TYPES)}")
return models
def load_tasks(path: Path = TASKS) -> pd.DataFrame:
df = pd.read_csv(path, dtype=str, keep_default_na=False)
if bad := [u for u in df["url"] if not is_https(u) or "#" in u]:
raise ValueError(f"{path.name}: not https:// links without #: {bad[:3]}")
df["horizon"] = df["horizon"].astype(int)
df["source"] = [with_text(u, p) for u, p in zip(df["url"], df["provider"], strict=True)]
return df.sort_values(["domain", "name"])
def name(model_id: str, models: Models) -> str:
if model_id == BASELINE:
return BASELINE_NAME
return models.get(model_id, {}).get("name") or model_id
def link(model_id: str, models: Models, paper_url: str | None = None) -> str:
return models.get(model_id, {}).get("url") or paper_url or ABOUT_PAGE
def is_baseline(model_id: str, models: Models) -> bool:
return model_id == BASELINE or bool(models.get(model_id, {}).get("baseline"))
def kind(model_id: str, models: Models) -> Kind:
return MODEL_TYPES.get(models.get(model_id, {}).get("type"), OTHER)
def shown(rows: list[dict[str, Any]], models: Models, types: set[str]) -> list[dict[str, Any]]:
def keep(row: dict[str, Any]) -> bool:
kind = models.get(row["model_id"], {}).get("type")
return row["model_id"] == BASELINE or kind is None or kind in types
return [row for row in rows if keep(row)]
def label(key: str) -> str:
return key.replace("_", " ")
def group_label(value: str) -> str:
return GROUP_LABELS.get(value) or label(value).capitalize()
def unique_labels(models: Iterable[tuple[str, str]]) -> dict[str, str]:
pairs = list(models)
counts = Counter(name for _, name in pairs)
return {mid: name if counts[name] == 1 else f"{name} ({mid})" for mid, name in pairs}
def utc(timestamp: str) -> str:
return f"{datetime.fromisoformat(timestamp).astimezone(UTC):%Y-%m-%d %H:%M} UTC"
def table_height(rows: int) -> int:
return (min(rows, TABLE_ROWS) + 1) * ROW_HEIGHT + 3
def overall(board: dict[str, Any]) -> dict[str, Any]:
return board["boards"].get("overall", EMPTY_BOARD)
def group_values(board: dict[str, Any], group_type: str) -> list[str]:
prefix = f"{group_type}/"
values = [k.removeprefix(prefix) for k in board["boards"] if k.startswith(prefix)]
if group_type == "freq":
order = {f: i for i, f in enumerate(FREQUENCIES)}
return sorted(values, key=lambda v: (order.get(v, len(order)), v))
return sorted(values)
def _percent(value: float | None) -> float | None:
return None if value is None else 100 * value
def marginal_win_rates(
rows: list[dict[str, Any]], metric: str, pairwise: dict[str, Any] | None
) -> dict[str, float | None]:
if not pairwise:
return {r["model_id"]: r["metrics"][metric]["marginal_win_rate"] for r in rows}
shown = {r["model_id"] for r in rows}
out = {}
for model_id in shown:
against = pairwise[metric].get(model_id, {})
rates = [
v["win_rate"] for o, v in against.items() if o in shown and v["win_rate"] is not None
]
out[model_id] = sum(rates) / len(rates) if rates else None
return out
def table(
rows: list[dict[str, Any]],
metric: str,
models: Models | None = None,
paper_url: str | None = None,
pairwise: dict[str, Any] | None = None,
samples: int = 0,
) -> pd.DataFrame:
models = models or {}
marginal = marginal_win_rates(rows, metric, pairwise)
def key(row: dict[str, Any]) -> tuple[bool, float]:
score = row["metrics"][metric]["skill_score"]
return score is not None, score or 0.0
records, place = [], 0
for row in sorted(rows, key=key, reverse=True):
model_id = row["model_id"]
prefix = ""
if not is_baseline(model_id, models):
place += 1
prefix = f"{MEDALS[place]} " if place in MEDALS else ""
records.append(
{
"model": (model := name(model_id, models)),
"link": with_text(link(model_id, models, paper_url), f"{prefix}{model}"),
"type": [MODEL_TYPES[t].label]
if (t := models.get(model_id, {}).get("type"))
else [],
"skill_score": _percent(row["metrics"][metric]["skill_score"]),
"win_rate": _percent(row["metrics"][metric]["win_rate"]),
"marginal_win_rate": _percent(marginal[model_id]),
"coverage_samples": 100 * (1 - row["missing_samples"] / samples)
if samples
else None,
"coverage_weight": _percent(
None if (w := row["missing_weight"]) is None else 1 - w
),
"model_id": model_id,
}
)
df = pd.DataFrame.from_records(records, columns=[*COLUMNS, *HIDDEN_COLUMNS])
return df if any(df["type"].map(len)) else df.drop(columns="type")
def competitors(df: pd.DataFrame, models: Models) -> pd.DataFrame:
return df.loc[~df["model_id"].map(lambda m: is_baseline(m, models)).astype(bool)]
def style(df: pd.DataFrame) -> Styler:
def colors(row: pd.Series) -> list[str]:
color = f"color: {BASELINE_COLOR}" if row["model_id"] == BASELINE else ""
return [color] * len(row)
return df.style.apply(colors, axis=1)
def breakdown(
board: dict[str, Any],
group_type: str,
metric: str,
columns: dict[str, str],
names: dict[str, str] | None = None,
) -> pd.DataFrame:
data = {}
for value in group_values(board, group_type):
rows = board["boards"][f"{group_type}/{value}"]["rankings"]
scores = {row["model_id"]: row["metrics"][metric]["skill_score"] for row in rows}
data[(names or {}).get(value) or group_label(value)] = [
float("nan") if scores.get(m) is None else 100 * scores[m] for m in columns
]
df = pd.DataFrame.from_dict(data, orient="index", columns=list(columns.values()), dtype=float)
return df.dropna(how="all")
def breakdown_style(df: pd.DataFrame) -> Styler:
def colors(row: pd.Series) -> list[str]:
ranks = row.rank(ascending=False, method="min")
return [
f"background-color: {RANK_COLORS[int(r)]}" if pd.notna(r) and r <= 3 else ""
for r in ranks
]
return df.style.apply(colors, axis=1).format("{:.1f}", na_rep="–")
INCIDENT = re.compile(r"^## (INC-[0-9]{4}-[0-9]{3}): (.+)$", re.MULTILINE)
INCIDENT_DATE = re.compile(r"^- \*\*Date:\*\* ([0-9]{4}-[0-9]{2}(?:-[0-9]{2})?)", re.MULTILINE)
IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
class Incident(NamedTuple):
id: str
title: str
date: str | None
text: str
def incidents(text: str | None) -> list[Incident]:
parts = INCIDENT.split(text or "")
found = []
for key, title, body in zip(parts[1::3], parts[2::3], parts[3::3], strict=True):
day = INCIDENT_DATE.search(body)
found.append(
Incident(key, title.strip(), day and day.group(1), IMAGE.sub("", body).strip())
)
return sorted(found, key=lambda i: i.id, reverse=True)