File size: 11,846 Bytes
585789a
 
 
 
 
 
 
 
 
b2c1aa1
585789a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2c1aa1
585789a
b2c1aa1
 
 
 
 
 
 
 
585789a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2c1aa1
 
 
 
 
 
 
585789a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2c1aa1
585789a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2c1aa1
585789a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2c1aa1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
from __future__ import annotations

import re
import tomllib
from collections import Counter
from collections.abc import Iterable
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, NamedTuple
from urllib.parse import quote, urlsplit

import pandas as pd
from pandas.io.formats.style import Styler

BASELINE = "__baseline__"
BASELINE_NAME = "Seasonal naive"
BREAKDOWNS = {"domain": "By domain", "freq": "By frequency", "task": "By task"}
FREQUENCIES = ("upto30min", "hourly", "daily", "weekly", "monthly")
GROUP_LABELS = {
    "gov_services": "Gov. Services",
    "public_interest": "Public Interest",
    "upto30min": "≤ 30 min",
}
EMPTY_BOARD: dict[str, Any] = {"sample_count": 0, "tasks": {}, "rankings": []}
METRICS = {"mase": "MASE", "sql": "SQL"}
VARIANTS = {"release": "Live", "all": "Every sample"}
VARIANT_HELP = (
    "A model's training data may leak a sample's actual values, directly or indirectly. Two "
    "modes allow both a strict and a faster comparison of models. Where a forecast doesn't "
    "count, the model gets the seasonal naive baseline's score.\n\n"
    "**Live:** only what a model could have forecast live. A pretrained model counts "
    "from its checkpoint's release, LLM-based and agentic models from their first live day.\n\n"
    "**Every sample:** pretrained models count on every sample (unless "
    "direct leakage is reported for specific samples; those fall back to the baseline), "
    "LLM-based models from their training-data cutoff, agentic models from their first live day."
)
METRIC_HELP = (
    "MASE: Mean Absolute Scaled Error (point forecast) "
    "SQL: Scaled Quantile Loss (probabilistic forecast)"
)


class Kind(NamedTuple):
    label: str
    tag: str
    bar: str


MODEL_TYPES = {
    "statistical": Kind("Statistical", "#ECE6DD", "#A08C6E"),
    "pretrained": Kind("Pretrained", "#E9DDF5", "#8D5EB7"),
    "llm-based": Kind("LLM-based", "#DCE6F2", "#4E79A7"),
    "agentic": Kind("Agentic", "#DCEEE4", "#3E9C6E"),
    "closed-api": Kind("Closed API", "#F3DEDE", "#C44E52"),
}
OTHER = Kind("Other", "#ECEDEF", "#9AA0A6")
TABLE_ROWS = 20
ROW_HEIGHT = 35
MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"}
BASELINE_COLOR = "#7D8998"
RANK_COLORS = {1: "#F7D36B", 2: "#E5E7EB", 3: "#E6B089"}

COLUMNS = {
    "link": "Model",
    "skill_score": "Skill score (%)",
    "win_rate": "Win rate (%)",
    "marginal_win_rate": "Marginal win rate (%)",
    "coverage_samples": "Coverage, samples (%)",
    "coverage_weight": "Coverage, weight (%)",
    "type": "Type",
}
HIDDEN_COLUMNS = ("model", "model_id")
TREND_VALUES = ("skill_score", "win_rate", "coverage_samples", "coverage_weight")
# The Model column links each name; its text is the URL's fragment.
LINK_TEXT = r"#(.*)$"


# The link column decodes its text after the "#" and reads "+" as a space.
def with_text(url: str, text: str) -> str:
    return f"{url}#{quote(text, safe='')}"


ABOUT_PAGE = "cref-about"

MODELS = Path(__file__).parent / "models.toml"
TASKS = Path(__file__).parent / "tasks.csv"
TASK_COLUMNS = {
    "name": "Task",
    "domain": "Domain",
    "frequency": "Frequency",
    "horizon": "Horizon (steps)",
    "source": "Source",
    "license": "License",
}
MODEL_FIELDS = {"name", "url", "type", "baseline"}
MODEL_ID = re.compile(rf"[a-z0-9](?:[a-z0-9-]{{0,62}}[a-z0-9])?|{BASELINE}")

Models = dict[str, dict[str, Any]]


def is_https(url: object) -> bool:
    if not isinstance(url, str) or any(c.isspace() or not c.isprintable() for c in url):
        return False
    parts = urlsplit(url)
    return parts.scheme == "https" and bool(parts.netloc)


def load_models(path: Path = MODELS) -> Models:
    models = tomllib.loads(path.read_text(encoding="utf-8"))
    for model_id, info in models.items():
        where = f"{path.name}: [{model_id}]"
        if not MODEL_ID.fullmatch(model_id) or not isinstance(info, dict):
            raise ValueError(f"{where}: not a model ID")
        if unknown := set(info) - MODEL_FIELDS:
            raise ValueError(f"{where}: unknown fields {sorted(unknown)}")
        if "url" in info and (not is_https(info["url"]) or "#" in info["url"]):
            raise ValueError(f"{where}: url is not an https:// link without #")
        if "baseline" in info and not isinstance(info["baseline"], bool):
            raise ValueError(f"{where}: baseline must be true or false")
        name = info.get("name", "x")
        if not isinstance(name, str) or not 0 < len(name) <= 256 or not name.isprintable():
            raise ValueError(f"{where}: name must be 1 to 256 printable characters")
        if "type" in info and info["type"] not in MODEL_TYPES:
            raise ValueError(f"{where}: type must be one of {list(MODEL_TYPES)}")
    return models


def load_tasks(path: Path = TASKS) -> pd.DataFrame:
    df = pd.read_csv(path, dtype=str, keep_default_na=False)
    if bad := [u for u in df["url"] if not is_https(u) or "#" in u]:
        raise ValueError(f"{path.name}: not https:// links without #: {bad[:3]}")
    df["horizon"] = df["horizon"].astype(int)
    df["source"] = [with_text(u, p) for u, p in zip(df["url"], df["provider"], strict=True)]
    return df.sort_values(["domain", "name"])


def name(model_id: str, models: Models) -> str:
    if model_id == BASELINE:
        return BASELINE_NAME
    return models.get(model_id, {}).get("name") or model_id


def link(model_id: str, models: Models, paper_url: str | None = None) -> str:
    return models.get(model_id, {}).get("url") or paper_url or ABOUT_PAGE


def is_baseline(model_id: str, models: Models) -> bool:
    return model_id == BASELINE or bool(models.get(model_id, {}).get("baseline"))


def kind(model_id: str, models: Models) -> Kind:
    return MODEL_TYPES.get(models.get(model_id, {}).get("type"), OTHER)


def shown(rows: list[dict[str, Any]], models: Models, types: set[str]) -> list[dict[str, Any]]:
    def keep(row: dict[str, Any]) -> bool:
        kind = models.get(row["model_id"], {}).get("type")
        return row["model_id"] == BASELINE or kind is None or kind in types

    return [row for row in rows if keep(row)]


def label(key: str) -> str:
    return key.replace("_", " ")


def group_label(value: str) -> str:
    return GROUP_LABELS.get(value) or label(value).capitalize()


def unique_labels(models: Iterable[tuple[str, str]]) -> dict[str, str]:
    pairs = list(models)
    counts = Counter(name for _, name in pairs)
    return {mid: name if counts[name] == 1 else f"{name} ({mid})" for mid, name in pairs}


def utc(timestamp: str) -> str:
    return f"{datetime.fromisoformat(timestamp).astimezone(UTC):%Y-%m-%d %H:%M} UTC"


def table_height(rows: int) -> int:
    return (min(rows, TABLE_ROWS) + 1) * ROW_HEIGHT + 3


def overall(board: dict[str, Any]) -> dict[str, Any]:
    return board["boards"].get("overall", EMPTY_BOARD)


def group_values(board: dict[str, Any], group_type: str) -> list[str]:
    prefix = f"{group_type}/"
    values = [k.removeprefix(prefix) for k in board["boards"] if k.startswith(prefix)]
    if group_type == "freq":
        order = {f: i for i, f in enumerate(FREQUENCIES)}
        return sorted(values, key=lambda v: (order.get(v, len(order)), v))
    return sorted(values)


def _percent(value: float | None) -> float | None:
    return None if value is None else 100 * value


def marginal_win_rates(
    rows: list[dict[str, Any]], metric: str, pairwise: dict[str, Any] | None
) -> dict[str, float | None]:
    if not pairwise:
        return {r["model_id"]: r["metrics"][metric]["marginal_win_rate"] for r in rows}
    shown = {r["model_id"] for r in rows}
    out = {}
    for model_id in shown:
        against = pairwise[metric].get(model_id, {})
        rates = [
            v["win_rate"] for o, v in against.items() if o in shown and v["win_rate"] is not None
        ]
        out[model_id] = sum(rates) / len(rates) if rates else None
    return out


def table(
    rows: list[dict[str, Any]],
    metric: str,
    models: Models | None = None,
    paper_url: str | None = None,
    pairwise: dict[str, Any] | None = None,
    samples: int = 0,
) -> pd.DataFrame:
    models = models or {}
    marginal = marginal_win_rates(rows, metric, pairwise)

    def key(row: dict[str, Any]) -> tuple[bool, float]:
        score = row["metrics"][metric]["skill_score"]
        return score is not None, score or 0.0

    records, place = [], 0
    for row in sorted(rows, key=key, reverse=True):
        model_id = row["model_id"]
        prefix = ""
        if not is_baseline(model_id, models):
            place += 1
            prefix = f"{MEDALS[place]} " if place in MEDALS else ""
        records.append(
            {
                "model": (model := name(model_id, models)),
                "link": with_text(link(model_id, models, paper_url), f"{prefix}{model}"),
                "type": [MODEL_TYPES[t].label]
                if (t := models.get(model_id, {}).get("type"))
                else [],
                "skill_score": _percent(row["metrics"][metric]["skill_score"]),
                "win_rate": _percent(row["metrics"][metric]["win_rate"]),
                "marginal_win_rate": _percent(marginal[model_id]),
                "coverage_samples": 100 * (1 - row["missing_samples"] / samples)
                if samples
                else None,
                "coverage_weight": _percent(
                    None if (w := row["missing_weight"]) is None else 1 - w
                ),
                "model_id": model_id,
            }
        )
    df = pd.DataFrame.from_records(records, columns=[*COLUMNS, *HIDDEN_COLUMNS])
    return df if any(df["type"].map(len)) else df.drop(columns="type")


def competitors(df: pd.DataFrame, models: Models) -> pd.DataFrame:
    return df.loc[~df["model_id"].map(lambda m: is_baseline(m, models)).astype(bool)]


def style(df: pd.DataFrame) -> Styler:
    def colors(row: pd.Series) -> list[str]:
        color = f"color: {BASELINE_COLOR}" if row["model_id"] == BASELINE else ""
        return [color] * len(row)

    return df.style.apply(colors, axis=1)


def breakdown(
    board: dict[str, Any],
    group_type: str,
    metric: str,
    columns: dict[str, str],
    names: dict[str, str] | None = None,
) -> pd.DataFrame:
    data = {}
    for value in group_values(board, group_type):
        rows = board["boards"][f"{group_type}/{value}"]["rankings"]
        scores = {row["model_id"]: row["metrics"][metric]["skill_score"] for row in rows}
        data[(names or {}).get(value) or group_label(value)] = [
            float("nan") if scores.get(m) is None else 100 * scores[m] for m in columns
        ]
    df = pd.DataFrame.from_dict(data, orient="index", columns=list(columns.values()), dtype=float)
    return df.dropna(how="all")


def breakdown_style(df: pd.DataFrame) -> Styler:
    def colors(row: pd.Series) -> list[str]:
        ranks = row.rank(ascending=False, method="min")
        return [
            f"background-color: {RANK_COLORS[int(r)]}" if pd.notna(r) and r <= 3 else ""
            for r in ranks
        ]

    return df.style.apply(colors, axis=1).format("{:.1f}", na_rep="–")


INCIDENT = re.compile(r"^## (INC-[0-9]{4}-[0-9]{3}): (.+)$", re.MULTILINE)
INCIDENT_DATE = re.compile(r"^- \*\*Date:\*\* ([0-9]{4}-[0-9]{2}(?:-[0-9]{2})?)", re.MULTILINE)
IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)")


class Incident(NamedTuple):
    id: str
    title: str
    date: str | None
    text: str


def incidents(text: str | None) -> list[Incident]:
    parts = INCIDENT.split(text or "")
    found = []
    for key, title, body in zip(parts[1::3], parts[2::3], parts[3::3], strict=True):
        day = INCIDENT_DATE.search(body)
        found.append(
            Incident(key, title.strip(), day and day.group(1), IMAGE.sub("", body).strip())
        )
    return sorted(found, key=lambda i: i.id, reverse=True)