from __future__ import annotations import re import tomllib from collections import Counter from collections.abc import Iterable from datetime import UTC, datetime from pathlib import Path from typing import Any, NamedTuple from urllib.parse import quote, urlsplit import pandas as pd from pandas.io.formats.style import Styler BASELINE = "__baseline__" BASELINE_NAME = "Seasonal naive" BREAKDOWNS = {"domain": "By domain", "freq": "By frequency", "task": "By task"} FREQUENCIES = ("upto30min", "hourly", "daily", "weekly", "monthly") GROUP_LABELS = { "gov_services": "Gov. Services", "public_interest": "Public Interest", "upto30min": "≤ 30 min", } EMPTY_BOARD: dict[str, Any] = {"sample_count": 0, "tasks": {}, "rankings": []} METRICS = {"mase": "MASE", "sql": "SQL"} VARIANTS = {"release": "Live", "all": "Every sample"} VARIANT_HELP = ( "A model's training data may leak a sample's actual values, directly or indirectly. Two " "modes allow both a strict and a faster comparison of models. Where a forecast doesn't " "count, the model gets the seasonal naive baseline's score.\n\n" "**Live:** only what a model could have forecast live. A pretrained model counts " "from its checkpoint's release, LLM-based and agentic models from their first live day.\n\n" "**Every sample:** pretrained models count on every sample (unless " "direct leakage is reported for specific samples; those fall back to the baseline), " "LLM-based models from their training-data cutoff, agentic models from their first live day." ) METRIC_HELP = ( "MASE: Mean Absolute Scaled Error (point forecast) " "SQL: Scaled Quantile Loss (probabilistic forecast)" ) class Kind(NamedTuple): label: str tag: str bar: str MODEL_TYPES = { "statistical": Kind("Statistical", "#ECE6DD", "#A08C6E"), "pretrained": Kind("Pretrained", "#E9DDF5", "#8D5EB7"), "llm-based": Kind("LLM-based", "#DCE6F2", "#4E79A7"), "agentic": Kind("Agentic", "#DCEEE4", "#3E9C6E"), "closed-api": Kind("Closed API", "#F3DEDE", "#C44E52"), } OTHER = Kind("Other", "#ECEDEF", "#9AA0A6") TABLE_ROWS = 20 ROW_HEIGHT = 35 MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"} BASELINE_COLOR = "#7D8998" RANK_COLORS = {1: "#F7D36B", 2: "#E5E7EB", 3: "#E6B089"} COLUMNS = { "link": "Model", "skill_score": "Skill score (%)", "win_rate": "Win rate (%)", "marginal_win_rate": "Marginal win rate (%)", "coverage_samples": "Coverage, samples (%)", "coverage_weight": "Coverage, weight (%)", "type": "Type", } HIDDEN_COLUMNS = ("model", "model_id") TREND_VALUES = ("skill_score", "win_rate", "coverage_samples", "coverage_weight") # The Model column links each name; its text is the URL's fragment. LINK_TEXT = r"#(.*)$" # The link column decodes its text after the "#" and reads "+" as a space. def with_text(url: str, text: str) -> str: return f"{url}#{quote(text, safe='')}" ABOUT_PAGE = "cref-about" MODELS = Path(__file__).parent / "models.toml" TASKS = Path(__file__).parent / "tasks.csv" TASK_COLUMNS = { "name": "Task", "domain": "Domain", "frequency": "Frequency", "horizon": "Horizon (steps)", "source": "Source", "license": "License", } MODEL_FIELDS = {"name", "url", "type", "baseline"} MODEL_ID = re.compile(rf"[a-z0-9](?:[a-z0-9-]{{0,62}}[a-z0-9])?|{BASELINE}") Models = dict[str, dict[str, Any]] def is_https(url: object) -> bool: if not isinstance(url, str) or any(c.isspace() or not c.isprintable() for c in url): return False parts = urlsplit(url) return parts.scheme == "https" and bool(parts.netloc) def load_models(path: Path = MODELS) -> Models: models = tomllib.loads(path.read_text(encoding="utf-8")) for model_id, info in models.items(): where = f"{path.name}: [{model_id}]" if not MODEL_ID.fullmatch(model_id) or not isinstance(info, dict): raise ValueError(f"{where}: not a model ID") if unknown := set(info) - MODEL_FIELDS: raise ValueError(f"{where}: unknown fields {sorted(unknown)}") if "url" in info and (not is_https(info["url"]) or "#" in info["url"]): raise ValueError(f"{where}: url is not an https:// link without #") if "baseline" in info and not isinstance(info["baseline"], bool): raise ValueError(f"{where}: baseline must be true or false") name = info.get("name", "x") if not isinstance(name, str) or not 0 < len(name) <= 256 or not name.isprintable(): raise ValueError(f"{where}: name must be 1 to 256 printable characters") if "type" in info and info["type"] not in MODEL_TYPES: raise ValueError(f"{where}: type must be one of {list(MODEL_TYPES)}") return models def load_tasks(path: Path = TASKS) -> pd.DataFrame: df = pd.read_csv(path, dtype=str, keep_default_na=False) if bad := [u for u in df["url"] if not is_https(u) or "#" in u]: raise ValueError(f"{path.name}: not https:// links without #: {bad[:3]}") df["horizon"] = df["horizon"].astype(int) df["source"] = [with_text(u, p) for u, p in zip(df["url"], df["provider"], strict=True)] return df.sort_values(["domain", "name"]) def name(model_id: str, models: Models) -> str: if model_id == BASELINE: return BASELINE_NAME return models.get(model_id, {}).get("name") or model_id def link(model_id: str, models: Models, paper_url: str | None = None) -> str: return models.get(model_id, {}).get("url") or paper_url or ABOUT_PAGE def is_baseline(model_id: str, models: Models) -> bool: return model_id == BASELINE or bool(models.get(model_id, {}).get("baseline")) def kind(model_id: str, models: Models) -> Kind: return MODEL_TYPES.get(models.get(model_id, {}).get("type"), OTHER) def shown(rows: list[dict[str, Any]], models: Models, types: set[str]) -> list[dict[str, Any]]: def keep(row: dict[str, Any]) -> bool: kind = models.get(row["model_id"], {}).get("type") return row["model_id"] == BASELINE or kind is None or kind in types return [row for row in rows if keep(row)] def label(key: str) -> str: return key.replace("_", " ") def group_label(value: str) -> str: return GROUP_LABELS.get(value) or label(value).capitalize() def unique_labels(models: Iterable[tuple[str, str]]) -> dict[str, str]: pairs = list(models) counts = Counter(name for _, name in pairs) return {mid: name if counts[name] == 1 else f"{name} ({mid})" for mid, name in pairs} def utc(timestamp: str) -> str: return f"{datetime.fromisoformat(timestamp).astimezone(UTC):%Y-%m-%d %H:%M} UTC" def table_height(rows: int) -> int: return (min(rows, TABLE_ROWS) + 1) * ROW_HEIGHT + 3 def overall(board: dict[str, Any]) -> dict[str, Any]: return board["boards"].get("overall", EMPTY_BOARD) def group_values(board: dict[str, Any], group_type: str) -> list[str]: prefix = f"{group_type}/" values = [k.removeprefix(prefix) for k in board["boards"] if k.startswith(prefix)] if group_type == "freq": order = {f: i for i, f in enumerate(FREQUENCIES)} return sorted(values, key=lambda v: (order.get(v, len(order)), v)) return sorted(values) def _percent(value: float | None) -> float | None: return None if value is None else 100 * value def marginal_win_rates( rows: list[dict[str, Any]], metric: str, pairwise: dict[str, Any] | None ) -> dict[str, float | None]: if not pairwise: return {r["model_id"]: r["metrics"][metric]["marginal_win_rate"] for r in rows} shown = {r["model_id"] for r in rows} out = {} for model_id in shown: against = pairwise[metric].get(model_id, {}) rates = [ v["win_rate"] for o, v in against.items() if o in shown and v["win_rate"] is not None ] out[model_id] = sum(rates) / len(rates) if rates else None return out def table( rows: list[dict[str, Any]], metric: str, models: Models | None = None, paper_url: str | None = None, pairwise: dict[str, Any] | None = None, samples: int = 0, ) -> pd.DataFrame: models = models or {} marginal = marginal_win_rates(rows, metric, pairwise) def key(row: dict[str, Any]) -> tuple[bool, float]: score = row["metrics"][metric]["skill_score"] return score is not None, score or 0.0 records, place = [], 0 for row in sorted(rows, key=key, reverse=True): model_id = row["model_id"] prefix = "" if not is_baseline(model_id, models): place += 1 prefix = f"{MEDALS[place]} " if place in MEDALS else "" records.append( { "model": (model := name(model_id, models)), "link": with_text(link(model_id, models, paper_url), f"{prefix}{model}"), "type": [MODEL_TYPES[t].label] if (t := models.get(model_id, {}).get("type")) else [], "skill_score": _percent(row["metrics"][metric]["skill_score"]), "win_rate": _percent(row["metrics"][metric]["win_rate"]), "marginal_win_rate": _percent(marginal[model_id]), "coverage_samples": 100 * (1 - row["missing_samples"] / samples) if samples else None, "coverage_weight": _percent( None if (w := row["missing_weight"]) is None else 1 - w ), "model_id": model_id, } ) df = pd.DataFrame.from_records(records, columns=[*COLUMNS, *HIDDEN_COLUMNS]) return df if any(df["type"].map(len)) else df.drop(columns="type") def competitors(df: pd.DataFrame, models: Models) -> pd.DataFrame: return df.loc[~df["model_id"].map(lambda m: is_baseline(m, models)).astype(bool)] def style(df: pd.DataFrame) -> Styler: def colors(row: pd.Series) -> list[str]: color = f"color: {BASELINE_COLOR}" if row["model_id"] == BASELINE else "" return [color] * len(row) return df.style.apply(colors, axis=1) def breakdown( board: dict[str, Any], group_type: str, metric: str, columns: dict[str, str], names: dict[str, str] | None = None, ) -> pd.DataFrame: data = {} for value in group_values(board, group_type): rows = board["boards"][f"{group_type}/{value}"]["rankings"] scores = {row["model_id"]: row["metrics"][metric]["skill_score"] for row in rows} data[(names or {}).get(value) or group_label(value)] = [ float("nan") if scores.get(m) is None else 100 * scores[m] for m in columns ] df = pd.DataFrame.from_dict(data, orient="index", columns=list(columns.values()), dtype=float) return df.dropna(how="all") def breakdown_style(df: pd.DataFrame) -> Styler: def colors(row: pd.Series) -> list[str]: ranks = row.rank(ascending=False, method="min") return [ f"background-color: {RANK_COLORS[int(r)]}" if pd.notna(r) and r <= 3 else "" for r in ranks ] return df.style.apply(colors, axis=1).format("{:.1f}", na_rep="–") INCIDENT = re.compile(r"^## (INC-[0-9]{4}-[0-9]{3}): (.+)$", re.MULTILINE) INCIDENT_DATE = re.compile(r"^- \*\*Date:\*\* ([0-9]{4}-[0-9]{2}(?:-[0-9]{2})?)", re.MULTILINE) IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)") class Incident(NamedTuple): id: str title: str date: str | None text: str def incidents(text: str | None) -> list[Incident]: parts = INCIDENT.split(text or "") found = [] for key, title, body in zip(parts[1::3], parts[2::3], parts[3::3], strict=True): day = INCIDENT_DATE.search(body) found.append( Incident(key, title.strip(), day and day.group(1), IMAGE.sub("", body).strip()) ) return sorted(found, key=lambda i: i.id, reverse=True)