Spaces:
Running
Running
Download cref/views.py from autogluon/fev-bench: direct link, hf CLI and curl.
- Browser
- Download file 11.8 kB
-
https://huggingface.co/spaces/autogluon/fev-bench/resolve/main/cref/views.py
- Command line
-
hf download hf://spaces/autogluon/fev-bench/cref/views.py
-
curl -L -o views.py https://huggingface.co/spaces/autogluon/fev-bench/resolve/main/cref/views.py
11.8 kB
| from __future__ import annotations | |
| import re | |
| import tomllib | |
| from collections import Counter | |
| from collections.abc import Iterable | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any, NamedTuple | |
| from urllib.parse import quote, urlsplit | |
| import pandas as pd | |
| from pandas.io.formats.style import Styler | |
| BASELINE = "__baseline__" | |
| BASELINE_NAME = "Seasonal naive" | |
| BREAKDOWNS = {"domain": "By domain", "freq": "By frequency", "task": "By task"} | |
| FREQUENCIES = ("upto30min", "hourly", "daily", "weekly", "monthly") | |
| GROUP_LABELS = { | |
| "gov_services": "Gov. Services", | |
| "public_interest": "Public Interest", | |
| "upto30min": "≤ 30 min", | |
| } | |
| EMPTY_BOARD: dict[str, Any] = {"sample_count": 0, "tasks": {}, "rankings": []} | |
| METRICS = {"mase": "MASE", "sql": "SQL"} | |
| VARIANTS = {"release": "Live", "all": "Every sample"} | |
| VARIANT_HELP = ( | |
| "A model's training data may leak a sample's actual values, directly or indirectly. Two " | |
| "modes allow both a strict and a faster comparison of models. Where a forecast doesn't " | |
| "count, the model gets the seasonal naive baseline's score.\n\n" | |
| "**Live:** only what a model could have forecast live. A pretrained model counts " | |
| "from its checkpoint's release, LLM-based and agentic models from their first live day.\n\n" | |
| "**Every sample:** pretrained models count on every sample (unless " | |
| "direct leakage is reported for specific samples; those fall back to the baseline), " | |
| "LLM-based models from their training-data cutoff, agentic models from their first live day." | |
| ) | |
| METRIC_HELP = ( | |
| "MASE: Mean Absolute Scaled Error (point forecast) " | |
| "SQL: Scaled Quantile Loss (probabilistic forecast)" | |
| ) | |
| class Kind(NamedTuple): | |
| label: str | |
| tag: str | |
| bar: str | |
| MODEL_TYPES = { | |
| "statistical": Kind("Statistical", "#ECE6DD", "#A08C6E"), | |
| "pretrained": Kind("Pretrained", "#E9DDF5", "#8D5EB7"), | |
| "llm-based": Kind("LLM-based", "#DCE6F2", "#4E79A7"), | |
| "agentic": Kind("Agentic", "#DCEEE4", "#3E9C6E"), | |
| "closed-api": Kind("Closed API", "#F3DEDE", "#C44E52"), | |
| } | |
| OTHER = Kind("Other", "#ECEDEF", "#9AA0A6") | |
| TABLE_ROWS = 20 | |
| ROW_HEIGHT = 35 | |
| MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"} | |
| BASELINE_COLOR = "#7D8998" | |
| RANK_COLORS = {1: "#F7D36B", 2: "#E5E7EB", 3: "#E6B089"} | |
| COLUMNS = { | |
| "link": "Model", | |
| "skill_score": "Skill score (%)", | |
| "win_rate": "Win rate (%)", | |
| "marginal_win_rate": "Marginal win rate (%)", | |
| "coverage_samples": "Coverage, samples (%)", | |
| "coverage_weight": "Coverage, weight (%)", | |
| "type": "Type", | |
| } | |
| HIDDEN_COLUMNS = ("model", "model_id") | |
| TREND_VALUES = ("skill_score", "win_rate", "coverage_samples", "coverage_weight") | |
| # The Model column links each name; its text is the URL's fragment. | |
| LINK_TEXT = r"#(.*)$" | |
| # The link column decodes its text after the "#" and reads "+" as a space. | |
| def with_text(url: str, text: str) -> str: | |
| return f"{url}#{quote(text, safe='')}" | |
| ABOUT_PAGE = "cref-about" | |
| MODELS = Path(__file__).parent / "models.toml" | |
| TASKS = Path(__file__).parent / "tasks.csv" | |
| TASK_COLUMNS = { | |
| "name": "Task", | |
| "domain": "Domain", | |
| "frequency": "Frequency", | |
| "horizon": "Horizon (steps)", | |
| "source": "Source", | |
| "license": "License", | |
| } | |
| MODEL_FIELDS = {"name", "url", "type", "baseline"} | |
| MODEL_ID = re.compile(rf"[a-z0-9](?:[a-z0-9-]{{0,62}}[a-z0-9])?|{BASELINE}") | |
| Models = dict[str, dict[str, Any]] | |
| def is_https(url: object) -> bool: | |
| if not isinstance(url, str) or any(c.isspace() or not c.isprintable() for c in url): | |
| return False | |
| parts = urlsplit(url) | |
| return parts.scheme == "https" and bool(parts.netloc) | |
| def load_models(path: Path = MODELS) -> Models: | |
| models = tomllib.loads(path.read_text(encoding="utf-8")) | |
| for model_id, info in models.items(): | |
| where = f"{path.name}: [{model_id}]" | |
| if not MODEL_ID.fullmatch(model_id) or not isinstance(info, dict): | |
| raise ValueError(f"{where}: not a model ID") | |
| if unknown := set(info) - MODEL_FIELDS: | |
| raise ValueError(f"{where}: unknown fields {sorted(unknown)}") | |
| if "url" in info and (not is_https(info["url"]) or "#" in info["url"]): | |
| raise ValueError(f"{where}: url is not an https:// link without #") | |
| if "baseline" in info and not isinstance(info["baseline"], bool): | |
| raise ValueError(f"{where}: baseline must be true or false") | |
| name = info.get("name", "x") | |
| if not isinstance(name, str) or not 0 < len(name) <= 256 or not name.isprintable(): | |
| raise ValueError(f"{where}: name must be 1 to 256 printable characters") | |
| if "type" in info and info["type"] not in MODEL_TYPES: | |
| raise ValueError(f"{where}: type must be one of {list(MODEL_TYPES)}") | |
| return models | |
| def load_tasks(path: Path = TASKS) -> pd.DataFrame: | |
| df = pd.read_csv(path, dtype=str, keep_default_na=False) | |
| if bad := [u for u in df["url"] if not is_https(u) or "#" in u]: | |
| raise ValueError(f"{path.name}: not https:// links without #: {bad[:3]}") | |
| df["horizon"] = df["horizon"].astype(int) | |
| df["source"] = [with_text(u, p) for u, p in zip(df["url"], df["provider"], strict=True)] | |
| return df.sort_values(["domain", "name"]) | |
| def name(model_id: str, models: Models) -> str: | |
| if model_id == BASELINE: | |
| return BASELINE_NAME | |
| return models.get(model_id, {}).get("name") or model_id | |
| def link(model_id: str, models: Models, paper_url: str | None = None) -> str: | |
| return models.get(model_id, {}).get("url") or paper_url or ABOUT_PAGE | |
| def is_baseline(model_id: str, models: Models) -> bool: | |
| return model_id == BASELINE or bool(models.get(model_id, {}).get("baseline")) | |
| def kind(model_id: str, models: Models) -> Kind: | |
| return MODEL_TYPES.get(models.get(model_id, {}).get("type"), OTHER) | |
| def shown(rows: list[dict[str, Any]], models: Models, types: set[str]) -> list[dict[str, Any]]: | |
| def keep(row: dict[str, Any]) -> bool: | |
| kind = models.get(row["model_id"], {}).get("type") | |
| return row["model_id"] == BASELINE or kind is None or kind in types | |
| return [row for row in rows if keep(row)] | |
| def label(key: str) -> str: | |
| return key.replace("_", " ") | |
| def group_label(value: str) -> str: | |
| return GROUP_LABELS.get(value) or label(value).capitalize() | |
| def unique_labels(models: Iterable[tuple[str, str]]) -> dict[str, str]: | |
| pairs = list(models) | |
| counts = Counter(name for _, name in pairs) | |
| return {mid: name if counts[name] == 1 else f"{name} ({mid})" for mid, name in pairs} | |
| def utc(timestamp: str) -> str: | |
| return f"{datetime.fromisoformat(timestamp).astimezone(UTC):%Y-%m-%d %H:%M} UTC" | |
| def table_height(rows: int) -> int: | |
| return (min(rows, TABLE_ROWS) + 1) * ROW_HEIGHT + 3 | |
| def overall(board: dict[str, Any]) -> dict[str, Any]: | |
| return board["boards"].get("overall", EMPTY_BOARD) | |
| def group_values(board: dict[str, Any], group_type: str) -> list[str]: | |
| prefix = f"{group_type}/" | |
| values = [k.removeprefix(prefix) for k in board["boards"] if k.startswith(prefix)] | |
| if group_type == "freq": | |
| order = {f: i for i, f in enumerate(FREQUENCIES)} | |
| return sorted(values, key=lambda v: (order.get(v, len(order)), v)) | |
| return sorted(values) | |
| def _percent(value: float | None) -> float | None: | |
| return None if value is None else 100 * value | |
| def marginal_win_rates( | |
| rows: list[dict[str, Any]], metric: str, pairwise: dict[str, Any] | None | |
| ) -> dict[str, float | None]: | |
| if not pairwise: | |
| return {r["model_id"]: r["metrics"][metric]["marginal_win_rate"] for r in rows} | |
| shown = {r["model_id"] for r in rows} | |
| out = {} | |
| for model_id in shown: | |
| against = pairwise[metric].get(model_id, {}) | |
| rates = [ | |
| v["win_rate"] for o, v in against.items() if o in shown and v["win_rate"] is not None | |
| ] | |
| out[model_id] = sum(rates) / len(rates) if rates else None | |
| return out | |
| def table( | |
| rows: list[dict[str, Any]], | |
| metric: str, | |
| models: Models | None = None, | |
| paper_url: str | None = None, | |
| pairwise: dict[str, Any] | None = None, | |
| samples: int = 0, | |
| ) -> pd.DataFrame: | |
| models = models or {} | |
| marginal = marginal_win_rates(rows, metric, pairwise) | |
| def key(row: dict[str, Any]) -> tuple[bool, float]: | |
| score = row["metrics"][metric]["skill_score"] | |
| return score is not None, score or 0.0 | |
| records, place = [], 0 | |
| for row in sorted(rows, key=key, reverse=True): | |
| model_id = row["model_id"] | |
| prefix = "" | |
| if not is_baseline(model_id, models): | |
| place += 1 | |
| prefix = f"{MEDALS[place]} " if place in MEDALS else "" | |
| records.append( | |
| { | |
| "model": (model := name(model_id, models)), | |
| "link": with_text(link(model_id, models, paper_url), f"{prefix}{model}"), | |
| "type": [MODEL_TYPES[t].label] | |
| if (t := models.get(model_id, {}).get("type")) | |
| else [], | |
| "skill_score": _percent(row["metrics"][metric]["skill_score"]), | |
| "win_rate": _percent(row["metrics"][metric]["win_rate"]), | |
| "marginal_win_rate": _percent(marginal[model_id]), | |
| "coverage_samples": 100 * (1 - row["missing_samples"] / samples) | |
| if samples | |
| else None, | |
| "coverage_weight": _percent( | |
| None if (w := row["missing_weight"]) is None else 1 - w | |
| ), | |
| "model_id": model_id, | |
| } | |
| ) | |
| df = pd.DataFrame.from_records(records, columns=[*COLUMNS, *HIDDEN_COLUMNS]) | |
| return df if any(df["type"].map(len)) else df.drop(columns="type") | |
| def competitors(df: pd.DataFrame, models: Models) -> pd.DataFrame: | |
| return df.loc[~df["model_id"].map(lambda m: is_baseline(m, models)).astype(bool)] | |
| def style(df: pd.DataFrame) -> Styler: | |
| def colors(row: pd.Series) -> list[str]: | |
| color = f"color: {BASELINE_COLOR}" if row["model_id"] == BASELINE else "" | |
| return [color] * len(row) | |
| return df.style.apply(colors, axis=1) | |
| def breakdown( | |
| board: dict[str, Any], | |
| group_type: str, | |
| metric: str, | |
| columns: dict[str, str], | |
| names: dict[str, str] | None = None, | |
| ) -> pd.DataFrame: | |
| data = {} | |
| for value in group_values(board, group_type): | |
| rows = board["boards"][f"{group_type}/{value}"]["rankings"] | |
| scores = {row["model_id"]: row["metrics"][metric]["skill_score"] for row in rows} | |
| data[(names or {}).get(value) or group_label(value)] = [ | |
| float("nan") if scores.get(m) is None else 100 * scores[m] for m in columns | |
| ] | |
| df = pd.DataFrame.from_dict(data, orient="index", columns=list(columns.values()), dtype=float) | |
| return df.dropna(how="all") | |
| def breakdown_style(df: pd.DataFrame) -> Styler: | |
| def colors(row: pd.Series) -> list[str]: | |
| ranks = row.rank(ascending=False, method="min") | |
| return [ | |
| f"background-color: {RANK_COLORS[int(r)]}" if pd.notna(r) and r <= 3 else "" | |
| for r in ranks | |
| ] | |
| return df.style.apply(colors, axis=1).format("{:.1f}", na_rep="–") | |
| INCIDENT = re.compile(r"^## (INC-[0-9]{4}-[0-9]{3}): (.+)$", re.MULTILINE) | |
| INCIDENT_DATE = re.compile(r"^- \*\*Date:\*\* ([0-9]{4}-[0-9]{2}(?:-[0-9]{2})?)", re.MULTILINE) | |
| IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)") | |
| class Incident(NamedTuple): | |
| id: str | |
| title: str | |
| date: str | None | |
| text: str | |
| def incidents(text: str | None) -> list[Incident]: | |
| parts = INCIDENT.split(text or "") | |
| found = [] | |
| for key, title, body in zip(parts[1::3], parts[2::3], parts[3::3], strict=True): | |
| day = INCIDENT_DATE.search(body) | |
| found.append( | |
| Incident(key, title.strip(), day and day.group(1), IMAGE.sub("", body).strip()) | |
| ) | |
| return sorted(found, key=lambda i: i.id, reverse=True) | |