Spaces:
Running
Running
Leaderboard, win-rate matrix, target data tab
Browse files
README.md
CHANGED
|
@@ -6,9 +6,9 @@ colorTo: gray
|
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
| 8 |
license: apache-2.0
|
| 9 |
-
short_description: Backtest
|
| 10 |
---
|
| 11 |
|
| 12 |
-
Dashboard for [open-predico](https://github.com/arnaud-vs/open-predico).
|
| 13 |
|
| 14 |
Streamlit app run in the browser with [stlite](https://github.com/whitphx/stlite) (static Space, no server). Locally: `pip install -r requirements.txt && streamlit run app.py`.
|
|
|
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
| 8 |
license: apache-2.0
|
| 9 |
+
short_description: Backtest leaderboard and target data for open-predico
|
| 10 |
---
|
| 11 |
|
| 12 |
+
Dashboard for [open-predico](https://github.com/arnaud-vs/open-predico). Backtest leaderboard, win rates and error breakdowns, plus an exploration of the target series. Reads the public dataset [open-predico/benchmark](https://huggingface.co/datasets/open-predico/benchmark).
|
| 13 |
|
| 14 |
Streamlit app run in the browser with [stlite](https://github.com/whitphx/stlite) (static Space, no server). Locally: `pip install -r requirements.txt && streamlit run app.py`.
|
app.py
CHANGED
|
@@ -1,9 +1,11 @@
|
|
| 1 |
-
"""open-predico dashboard: backtest scores
|
| 2 |
|
| 3 |
Runs in the browser with stlite (Streamlit on Pyodide, see index.html), so the Space is a free
|
| 4 |
-
static one; `streamlit run app.py` works locally too.
|
|
|
|
| 5 |
"""
|
| 6 |
|
|
|
|
| 7 |
import io
|
| 8 |
import json
|
| 9 |
import sys
|
|
@@ -18,6 +20,20 @@ HF = "https://huggingface.co"
|
|
| 18 |
PREFIX = "predictions/backtest/"
|
| 19 |
TZ = "Europe/Madrid"
|
| 20 |
ALPHA = 0.2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
COLOR = alt.Color(
|
| 23 |
"config:N", scale=alt.Scale(scheme="tableau10"), legend=alt.Legend(orient="bottom")
|
|
@@ -44,9 +60,20 @@ def fetch(url: str) -> bytes:
|
|
| 44 |
return req.response.to_py().tobytes()
|
| 45 |
|
| 46 |
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
tree = json.loads(
|
| 51 |
fetch(f"{HF}/api/datasets/{REPO}/tree/main/{PREFIX.rstrip('/')}?recursive=true")
|
| 52 |
)
|
|
@@ -54,7 +81,7 @@ def load() -> pd.DataFrame:
|
|
| 54 |
out = []
|
| 55 |
for f in files:
|
| 56 |
config, target = f.removeprefix(PREFIX).removesuffix(".parquet").split("/")
|
| 57 |
-
df =
|
| 58 |
# Same formulas as src/openpredico/evaluation/metrics.py (copied, keep in sync).
|
| 59 |
o, q10, q50, q90 = df.obs, df.q10, df.q50, df.q90
|
| 60 |
df["se"] = (o - q50) ** 2
|
|
@@ -70,14 +97,15 @@ def load() -> pd.DataFrame:
|
|
| 70 |
df["day"] = local.tz_localize(None).normalize()
|
| 71 |
df["hour"] = local.hour
|
| 72 |
df["weekday"] = local.dayofweek
|
| 73 |
-
df["config"], df["target"] = config, target
|
| 74 |
out.append(df.reset_index())
|
| 75 |
return pd.concat(out, ignore_index=True)
|
| 76 |
|
| 77 |
|
| 78 |
-
|
|
|
|
| 79 |
"""Daily scores; a day with any missing truth is skipped, as Predico does."""
|
| 80 |
-
g =
|
| 81 |
d = g.agg(
|
| 82 |
se=("se", "mean"),
|
| 83 |
winkler=("winkler", "mean"),
|
|
@@ -88,10 +116,91 @@ def daily(steps: pd.DataFrame) -> pd.DataFrame:
|
|
| 88 |
return d[~d.pop("missing")].reset_index()
|
| 89 |
|
| 90 |
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
|
| 96 |
st.title("open-predico")
|
| 97 |
st.caption(
|
|
@@ -99,97 +208,135 @@ st.caption(
|
|
| 99 |
"[Code](https://github.com/arnaud-vs/open-predico) · "
|
| 100 |
"[Data](https://huggingface.co/datasets/open-predico/benchmark)"
|
| 101 |
)
|
| 102 |
-
|
| 103 |
|
| 104 |
-
with
|
|
|
|
|
|
|
| 105 |
st.markdown(
|
| 106 |
-
"Test year 1 Nov 2025 – 31 Oct 2026, forecast made at 14:00 Madrid on D-1
|
| 107 |
-
"
|
| 108 |
-
"Days with any missing truth are skipped, like Predico."
|
| 109 |
)
|
| 110 |
|
| 111 |
st.subheader("Leaderboard")
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
)
|
| 123 |
-
.sort_values("rmse")
|
| 124 |
)
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
|
|
|
| 128 |
)
|
|
|
|
| 129 |
|
| 130 |
st.subheader("By month")
|
| 131 |
-
target = st.
|
| 132 |
monthly = (
|
| 133 |
days[days.target == target]
|
| 134 |
.assign(month=lambda d: d.day.dt.to_period("M").dt.to_timestamp())
|
| 135 |
-
.groupby(["config", "month"])[
|
| 136 |
.mean()
|
| 137 |
.reset_index()
|
| 138 |
)
|
| 139 |
cols = st.columns(2)
|
| 140 |
-
for col,
|
| 141 |
col.altair_chart(
|
| 142 |
-
alt.Chart(monthly, title=
|
| 143 |
.mark_line(point=True)
|
| 144 |
.encode(
|
| 145 |
x=alt.X("yearmonth(month):T", title=None),
|
| 146 |
-
y=alt.Y(f"{
|
| 147 |
color=COLOR,
|
| 148 |
-
tooltip=["config", "yearmonth(month):T", alt.Tooltip(f"{
|
| 149 |
),
|
| 150 |
)
|
| 151 |
|
| 152 |
-
st.subheader("Paired comparison")
|
| 153 |
-
c1, c2 = st.columns(2)
|
| 154 |
-
a = c1.selectbox("Config A", configs, index=len(configs) - 1)
|
| 155 |
-
b = c2.selectbox("Config B", configs, index=0)
|
| 156 |
-
rows = []
|
| 157 |
-
for t in targets:
|
| 158 |
-
p = days[days.target == t].pivot(index="day", columns="config", values=["rmse", "winkler"])
|
| 159 |
-
both = p.dropna()
|
| 160 |
-
rows.append(
|
| 161 |
-
{
|
| 162 |
-
"target": t,
|
| 163 |
-
"days": len(both),
|
| 164 |
-
"A wins RMSE": (both["rmse"][a] < both["rmse"][b]).mean(),
|
| 165 |
-
"A wins Winkler": (both["winkler"][a] < both["winkler"][b]).mean(),
|
| 166 |
-
}
|
| 167 |
-
)
|
| 168 |
-
st.dataframe(
|
| 169 |
-
pd.DataFrame(rows)
|
| 170 |
-
.set_index("target")
|
| 171 |
-
.style.format({"A wins RMSE": "{:.0%}", "A wins Winkler": "{:.0%}"}),
|
| 172 |
-
)
|
| 173 |
-
|
| 174 |
st.subheader("Absolute error of q50 by hour and weekday (Madrid time)")
|
| 175 |
-
target = st.
|
| 176 |
sel = steps[steps.target == target]
|
| 177 |
cols = st.columns(2)
|
| 178 |
for col, key, title in [(cols[0], "hour", "Hour of day"), (cols[1], "weekday", "Weekday")]:
|
| 179 |
prof = sel.groupby(["config", key]).ae.mean().reset_index()
|
| 180 |
if key == "weekday":
|
| 181 |
-
prof["weekday"] = prof.weekday.map(
|
| 182 |
-
dict(enumerate("Mon Tue Wed Thu Fri Sat Sun".split()))
|
| 183 |
-
)
|
| 184 |
col.altair_chart(
|
| 185 |
alt.Chart(prof, title=title)
|
| 186 |
.mark_line(point=True)
|
| 187 |
.encode(
|
| 188 |
-
x=alt.X(
|
| 189 |
-
f"{key}:O",
|
| 190 |
-
title=None,
|
| 191 |
-
sort="Mon Tue Wed Thu Fri Sat Sun".split() if key == "weekday" else None,
|
| 192 |
-
),
|
| 193 |
y=alt.Y("ae:Q", title="MAE"),
|
| 194 |
color=COLOR,
|
| 195 |
tooltip=["config", key, alt.Tooltip("ae:Q", format=".1f")],
|
|
@@ -198,21 +345,22 @@ with tab:
|
|
| 198 |
|
| 199 |
st.subheader("Day viewer")
|
| 200 |
c1, c2 = st.columns(2)
|
| 201 |
-
target = c1.
|
| 202 |
lo, hi = steps.day.min().date(), steps.day.max().date()
|
| 203 |
date = c2.date_input("Day (Madrid)", value=hi, min_value=lo, max_value=hi)
|
| 204 |
-
|
|
|
|
|
|
|
|
|
|
| 205 |
view["time"] = view.time.dt.tz_convert(TZ).dt.tz_localize(None)
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
shown = days[(days.target == target) & (days.day == pd.Timestamp(date))]
|
| 209 |
-
if shown.empty:
|
| 210 |
st.caption("Truth incomplete on this day: not scored.")
|
| 211 |
else:
|
| 212 |
st.caption(
|
| 213 |
" · ".join(
|
| 214 |
f"{r.config}: RMSE {r.rmse:.1f}, Winkler {r.winkler:.1f}"
|
| 215 |
-
for r in
|
| 216 |
)
|
| 217 |
)
|
| 218 |
base = alt.Chart(view).encode(x=alt.X("time:T", title=None))
|
|
@@ -225,3 +373,137 @@ with tab:
|
|
| 225 |
)
|
| 226 |
st.altair_chart(band + q50 + obs)
|
| 227 |
st.caption("Shaded: q10–q90 band; solid: q50; dashed grey: truth.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""open-predico dashboard: backtest scores and target data from open-predico/benchmark (public).
|
| 2 |
|
| 3 |
Runs in the browser with stlite (Streamlit on Pyodide, see index.html), so the Space is a free
|
| 4 |
+
static one; `streamlit run app.py` works locally too. Pyodide is slow: every file is fetched once
|
| 5 |
+
and cached, and plots get aggregated data.
|
| 6 |
"""
|
| 7 |
|
| 8 |
+
import datetime as dt
|
| 9 |
import io
|
| 10 |
import json
|
| 11 |
import sys
|
|
|
|
| 20 |
PREFIX = "predictions/backtest/"
|
| 21 |
TZ = "Europe/Madrid"
|
| 22 |
ALPHA = 0.2
|
| 23 |
+
BASELINE = "slot_climatology_28d"
|
| 24 |
+
METRICS = {"rmse": "RMSE", "winkler": "Winkler"}
|
| 25 |
+
WEEKDAYS = "Mon Tue Wed Thu Fri Sat Sun".split()
|
| 26 |
+
# Model type from the config name prefix; anything else is an ML model.
|
| 27 |
+
TYPES = {
|
| 28 |
+
"slot_climatology": "baseline",
|
| 29 |
+
"persistence": "baseline",
|
| 30 |
+
"zero_inflated": "baseline",
|
| 31 |
+
"chronos": "foundation model",
|
| 32 |
+
"timesfm": "foundation model",
|
| 33 |
+
"timestral": "foundation model",
|
| 34 |
+
"t0": "foundation model",
|
| 35 |
+
"toto": "foundation model",
|
| 36 |
+
}
|
| 37 |
|
| 38 |
COLOR = alt.Color(
|
| 39 |
"config:N", scale=alt.Scale(scheme="tableau10"), legend=alt.Legend(orient="bottom")
|
|
|
|
| 60 |
return req.response.to_py().tobytes()
|
| 61 |
|
| 62 |
|
| 63 |
+
def parquet(path: str) -> pd.DataFrame:
|
| 64 |
+
return pd.read_parquet(io.BytesIO(fetch(f"{HF}/datasets/{REPO}/resolve/main/{path}")))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def model_type(config: str) -> str:
|
| 68 |
+
return next((t for p, t in TYPES.items() if config.startswith(p)), "ML")
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
# --- Backtest data ---------------------------------------------------------------------------
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@st.cache_resource(ttl=3600)
|
| 75 |
+
def load_steps() -> pd.DataFrame:
|
| 76 |
+
"""Per-step errors for every config folder under predictions/backtest/, with local time."""
|
| 77 |
tree = json.loads(
|
| 78 |
fetch(f"{HF}/api/datasets/{REPO}/tree/main/{PREFIX.rstrip('/')}?recursive=true")
|
| 79 |
)
|
|
|
|
| 81 |
out = []
|
| 82 |
for f in files:
|
| 83 |
config, target = f.removeprefix(PREFIX).removesuffix(".parquet").split("/")
|
| 84 |
+
df = parquet(f)
|
| 85 |
# Same formulas as src/openpredico/evaluation/metrics.py (copied, keep in sync).
|
| 86 |
o, q10, q50, q90 = df.obs, df.q10, df.q50, df.q90
|
| 87 |
df["se"] = (o - q50) ** 2
|
|
|
|
| 97 |
df["day"] = local.tz_localize(None).normalize()
|
| 98 |
df["hour"] = local.hour
|
| 99 |
df["weekday"] = local.dayofweek
|
| 100 |
+
df["config"], df["target"] = config, target.removeprefix("spain_mfrr_")
|
| 101 |
out.append(df.reset_index())
|
| 102 |
return pd.concat(out, ignore_index=True)
|
| 103 |
|
| 104 |
|
| 105 |
+
@st.cache_resource(ttl=3600)
|
| 106 |
+
def load_days() -> pd.DataFrame:
|
| 107 |
"""Daily scores; a day with any missing truth is skipped, as Predico does."""
|
| 108 |
+
g = load_steps().groupby(["target", "config", "day"])
|
| 109 |
d = g.agg(
|
| 110 |
se=("se", "mean"),
|
| 111 |
winkler=("winkler", "mean"),
|
|
|
|
| 116 |
return d[~d.pop("missing")].reset_index()
|
| 117 |
|
| 118 |
|
| 119 |
+
def leaderboard(days: pd.DataFrame) -> pd.DataFrame:
|
| 120 |
+
"""One row per config: raw and baseline-normalised scores per target, aggregate and rank.
|
| 121 |
+
|
| 122 |
+
Normalised = mean daily score / baseline's mean daily score, on days both are scored.
|
| 123 |
+
Score = geometric mean of the normalised RMSE and Winkler over both targets.
|
| 124 |
+
"""
|
| 125 |
+
cols = {}
|
| 126 |
+
for t, d in days.groupby("target"):
|
| 127 |
+
p = d.pivot(index="day", columns="config", values=["rmse", "winkler", "coverage"])
|
| 128 |
+
for m in METRICS:
|
| 129 |
+
cols[f"{t} {m}"] = p[m].mean()
|
| 130 |
+
if BASELINE in p[m]:
|
| 131 |
+
both = p[m].dropna(subset=[BASELINE])
|
| 132 |
+
cols[f"{t} {m} rel"] = both.mean() / both[BASELINE].mean()
|
| 133 |
+
cols[f"{t} coverage"] = p["coverage"].mean()
|
| 134 |
+
cols[f"{t} days"] = p["rmse"].count()
|
| 135 |
+
lb = pd.DataFrame(cols)
|
| 136 |
+
rel = lb.filter(like=" rel")
|
| 137 |
+
lb.insert(0, "score", np.exp(np.log(rel).mean(axis=1, skipna=False)))
|
| 138 |
+
lb.insert(0, "type", [model_type(c) for c in lb.index])
|
| 139 |
+
lb.insert(0, "rank", lb.score.rank(method="min"))
|
| 140 |
+
return lb.sort_values(["score", "rank"]).rename_axis("config").reset_index()
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def win_rates(days: pd.DataFrame, target: str, metric: str, order: list[str]) -> pd.DataFrame:
|
| 144 |
+
"""Long frame: share of commonly scored days where row config A beats column config B."""
|
| 145 |
+
p = days[days.target == target].pivot(index="day", columns="config", values=metric)
|
| 146 |
+
rows = []
|
| 147 |
+
for a in order:
|
| 148 |
+
for b in order:
|
| 149 |
+
both = p[[a, b]].dropna() if a != b else None
|
| 150 |
+
rows.append(
|
| 151 |
+
{
|
| 152 |
+
"A": a,
|
| 153 |
+
"B": b,
|
| 154 |
+
"win": (both[a] < both[b]).mean() if a != b and len(both) else np.nan,
|
| 155 |
+
"days": len(both) if a != b else 0,
|
| 156 |
+
}
|
| 157 |
+
)
|
| 158 |
+
return pd.DataFrame(rows)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
# --- Target data -----------------------------------------------------------------------------
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
@st.cache_resource
|
| 165 |
+
def load_target() -> pd.DataFrame:
|
| 166 |
+
"""v1/target.parquet in long form with local calendar columns."""
|
| 167 |
+
df = parquet("v1/target.parquet")
|
| 168 |
+
local = df.index.tz_convert(TZ)
|
| 169 |
+
cal = pd.DataFrame(
|
| 170 |
+
{
|
| 171 |
+
"time": local.tz_localize(None),
|
| 172 |
+
"day": local.tz_localize(None).normalize(),
|
| 173 |
+
"slot": local.strftime("%H:%M"),
|
| 174 |
+
"weekday": local.dayofweek,
|
| 175 |
+
"month": local.tz_localize(None).to_period("M").to_timestamp(),
|
| 176 |
+
}
|
| 177 |
+
)
|
| 178 |
+
out = []
|
| 179 |
+
for col in df.columns:
|
| 180 |
+
out.append(cal.assign(target=col.removeprefix("mfrr_"), value=df[col].to_numpy()))
|
| 181 |
+
return pd.concat(out, ignore_index=True)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
@st.cache_resource
|
| 185 |
+
def profile(target: str, key: str) -> pd.DataFrame:
|
| 186 |
+
"""Mean and q10/q90 of the target by a calendar key."""
|
| 187 |
+
v = load_target()
|
| 188 |
+
g = v[v.target == target].groupby(key).value
|
| 189 |
+
return pd.DataFrame(
|
| 190 |
+
{"mean": g.mean(), "q10": g.quantile(0.1), "q90": g.quantile(0.9)}
|
| 191 |
+
).reset_index()
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def band_chart(df: pd.DataFrame, x: alt.X, title: str) -> alt.LayerChart:
|
| 195 |
+
base = alt.Chart(df, title=title).encode(x=x)
|
| 196 |
+
band = base.mark_area(opacity=0.25).encode(y=alt.Y("q10:Q", title=None), y2="q90:Q")
|
| 197 |
+
line = base.mark_line(point=len(df) < 40).encode(
|
| 198 |
+
y="mean:Q", tooltip=[x.shorthand, alt.Tooltip("mean:Q", format=".1f")]
|
| 199 |
+
)
|
| 200 |
+
return band + line
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
# --- Page ------------------------------------------------------------------------------------
|
| 204 |
|
| 205 |
st.title("open-predico")
|
| 206 |
st.caption(
|
|
|
|
| 208 |
"[Code](https://github.com/arnaud-vs/open-predico) · "
|
| 209 |
"[Data](https://huggingface.co/datasets/open-predico/benchmark)"
|
| 210 |
)
|
| 211 |
+
tab_bt, tab_target = st.tabs(["Backtest", "Target data"])
|
| 212 |
|
| 213 |
+
with tab_bt:
|
| 214 |
+
steps, days = load_steps(), load_days()
|
| 215 |
+
targets = sorted(days.target.unique())
|
| 216 |
st.markdown(
|
| 217 |
+
"Test year 1 Nov 2025 – 31 Oct 2026, forecast made at 14:00 Madrid on D-1, scored per "
|
| 218 |
+
"local day; lower is better. Days with any missing truth are skipped, like Predico."
|
|
|
|
| 219 |
)
|
| 220 |
|
| 221 |
st.subheader("Leaderboard")
|
| 222 |
+
lb = leaderboard(days)
|
| 223 |
+
order = lb.config.tolist()
|
| 224 |
+
types = st.pills("Model type", sorted(lb.type.unique()), selection_mode="multi", default=None)
|
| 225 |
+
absolute = st.toggle("Show absolute scores")
|
| 226 |
+
shown = lb[lb.type.isin(types)] if types else lb
|
| 227 |
+
keep = ["rank", "config", "type", "score"]
|
| 228 |
+
for t in targets:
|
| 229 |
+
keep += [f"{t} rmse rel", f"{t} winkler rel", f"{t} coverage"]
|
| 230 |
+
keep += [f"{t} rmse", f"{t} winkler", f"{t} days"] if absolute else []
|
| 231 |
+
ratio = "Mean daily {} / baseline's, on days both are scored; < 1 beats the baseline."
|
| 232 |
+
colcfg = {
|
| 233 |
+
"rank": st.column_config.NumberColumn("Rank", format="%d"),
|
| 234 |
+
"score": st.column_config.NumberColumn(
|
| 235 |
+
"Score",
|
| 236 |
+
format="%.3f",
|
| 237 |
+
help="Geometric mean of the four normalised scores (RMSE and Winkler, both targets).",
|
| 238 |
+
),
|
| 239 |
+
}
|
| 240 |
+
for t in targets:
|
| 241 |
+
colcfg |= {
|
| 242 |
+
f"{t} rmse rel": st.column_config.NumberColumn(
|
| 243 |
+
f"{t} RMSE", format="%.3f", help=ratio.format("RMSE of q50")
|
| 244 |
+
),
|
| 245 |
+
f"{t} winkler rel": st.column_config.NumberColumn(
|
| 246 |
+
f"{t} Winkler", format="%.3f", help=ratio.format("Winkler score of q10/q90")
|
| 247 |
+
),
|
| 248 |
+
f"{t} coverage": st.column_config.NumberColumn(
|
| 249 |
+
f"{t} coverage",
|
| 250 |
+
format="percent",
|
| 251 |
+
help="Share of quarter-hours inside q10–q90; ideal 80%.",
|
| 252 |
+
),
|
| 253 |
+
f"{t} rmse": st.column_config.NumberColumn(f"{t} RMSE abs", format="%.1f"),
|
| 254 |
+
f"{t} winkler": st.column_config.NumberColumn(f"{t} Winkler abs", format="%.1f"),
|
| 255 |
+
f"{t} days": st.column_config.NumberColumn(f"{t} days", format="%d"),
|
| 256 |
+
}
|
| 257 |
+
st.dataframe(shown[keep], column_config=colcfg, hide_index=True)
|
| 258 |
+
with st.expander("Metrics"):
|
| 259 |
+
st.markdown(
|
| 260 |
+
f"""
|
| 261 |
+
- **RMSE** (Predico's q50 score): `sqrt(mean_t (obs − q50)²)` over a local day.
|
| 262 |
+
- **Winkler** (Predico's q10/q90 score, α = 0.2): `mean_t [(q90 − q10) + 10·max(0, q10 − obs)
|
| 263 |
+
+ 10·max(0, obs − q90)]`. Rewards narrow intervals, penalises misses.
|
| 264 |
+
- **Coverage**: share of quarter-hours with obs inside [q10, q90]; a calibrated model gets 80%.
|
| 265 |
+
- **Normalised**: a config's mean daily score divided by the baseline `{BASELINE}` (= 1.0) on
|
| 266 |
+
the days both are scored. **Score** is the geometric mean of the four normalised values;
|
| 267 |
+
**Rank** sorts by it.
|
| 268 |
+
- **Type**: baseline, foundation model (pretrained, zero-shot), ML (trained here).
|
| 269 |
+
"""
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
st.subheader("Win rate")
|
| 273 |
+
st.caption(
|
| 274 |
+
"Cell (row A, column B): share of days scored for both where A has the lower score. "
|
| 275 |
+
"Read across a row: green = A usually wins."
|
| 276 |
+
)
|
| 277 |
+
target = st.radio("Target", targets, horizontal=True, key="win_target")
|
| 278 |
+
cols = st.columns(2)
|
| 279 |
+
for col, (m, label) in zip(cols, METRICS.items(), strict=True):
|
| 280 |
+
w = win_rates(days, target, m, order)
|
| 281 |
+
enc = {
|
| 282 |
+
"x": alt.X("B:N", sort=order, title="B"),
|
| 283 |
+
"y": alt.Y("A:N", sort=order, title="A"),
|
| 284 |
+
}
|
| 285 |
+
heat = (
|
| 286 |
+
alt.Chart(w, title=label)
|
| 287 |
+
.mark_rect()
|
| 288 |
+
.encode(
|
| 289 |
+
**enc,
|
| 290 |
+
color=alt.Color(
|
| 291 |
+
"win:Q",
|
| 292 |
+
scale=alt.Scale(scheme="redyellowgreen", domain=[0, 1]),
|
| 293 |
+
legend=alt.Legend(format="%", title="A wins"),
|
| 294 |
+
),
|
| 295 |
+
tooltip=["A", "B", alt.Tooltip("win:Q", format=".0%"), "days"],
|
| 296 |
)
|
|
|
|
| 297 |
)
|
| 298 |
+
text = (
|
| 299 |
+
alt.Chart(w)
|
| 300 |
+
.mark_text(color="black")
|
| 301 |
+
.encode(**enc, text=alt.Text("win:Q", format=".0%"))
|
| 302 |
)
|
| 303 |
+
col.altair_chart(heat + text)
|
| 304 |
|
| 305 |
st.subheader("By month")
|
| 306 |
+
target = st.radio("Target", targets, horizontal=True, key="month_target")
|
| 307 |
monthly = (
|
| 308 |
days[days.target == target]
|
| 309 |
.assign(month=lambda d: d.day.dt.to_period("M").dt.to_timestamp())
|
| 310 |
+
.groupby(["config", "month"])[list(METRICS)]
|
| 311 |
.mean()
|
| 312 |
.reset_index()
|
| 313 |
)
|
| 314 |
cols = st.columns(2)
|
| 315 |
+
for col, (m, label) in zip(cols, METRICS.items(), strict=True):
|
| 316 |
col.altair_chart(
|
| 317 |
+
alt.Chart(monthly, title=label)
|
| 318 |
.mark_line(point=True)
|
| 319 |
.encode(
|
| 320 |
x=alt.X("yearmonth(month):T", title=None),
|
| 321 |
+
y=alt.Y(f"{m}:Q", title=None),
|
| 322 |
color=COLOR,
|
| 323 |
+
tooltip=["config", "yearmonth(month):T", alt.Tooltip(f"{m}:Q", format=".1f")],
|
| 324 |
),
|
| 325 |
)
|
| 326 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
st.subheader("Absolute error of q50 by hour and weekday (Madrid time)")
|
| 328 |
+
target = st.radio("Target", targets, horizontal=True, key="hour_target")
|
| 329 |
sel = steps[steps.target == target]
|
| 330 |
cols = st.columns(2)
|
| 331 |
for col, key, title in [(cols[0], "hour", "Hour of day"), (cols[1], "weekday", "Weekday")]:
|
| 332 |
prof = sel.groupby(["config", key]).ae.mean().reset_index()
|
| 333 |
if key == "weekday":
|
| 334 |
+
prof["weekday"] = prof.weekday.map(dict(enumerate(WEEKDAYS)))
|
|
|
|
|
|
|
| 335 |
col.altair_chart(
|
| 336 |
alt.Chart(prof, title=title)
|
| 337 |
.mark_line(point=True)
|
| 338 |
.encode(
|
| 339 |
+
x=alt.X(f"{key}:O", title=None, sort=WEEKDAYS if key == "weekday" else None),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
y=alt.Y("ae:Q", title="MAE"),
|
| 341 |
color=COLOR,
|
| 342 |
tooltip=["config", key, alt.Tooltip("ae:Q", format=".1f")],
|
|
|
|
| 345 |
|
| 346 |
st.subheader("Day viewer")
|
| 347 |
c1, c2 = st.columns(2)
|
| 348 |
+
target = c1.radio("Target", targets, horizontal=True, key="day_target")
|
| 349 |
lo, hi = steps.day.min().date(), steps.day.max().date()
|
| 350 |
date = c2.date_input("Day (Madrid)", value=hi, min_value=lo, max_value=hi)
|
| 351 |
+
cfgs = st.multiselect("Configs", order, default=order)
|
| 352 |
+
view = steps[
|
| 353 |
+
(steps.target == target) & (steps.day == pd.Timestamp(date)) & steps.config.isin(cfgs)
|
| 354 |
+
].copy()
|
| 355 |
view["time"] = view.time.dt.tz_convert(TZ).dt.tz_localize(None)
|
| 356 |
+
scored = days[(days.target == target) & (days.day == pd.Timestamp(date))]
|
| 357 |
+
if scored.empty:
|
|
|
|
|
|
|
| 358 |
st.caption("Truth incomplete on this day: not scored.")
|
| 359 |
else:
|
| 360 |
st.caption(
|
| 361 |
" · ".join(
|
| 362 |
f"{r.config}: RMSE {r.rmse:.1f}, Winkler {r.winkler:.1f}"
|
| 363 |
+
for r in scored.itertuples()
|
| 364 |
)
|
| 365 |
)
|
| 366 |
base = alt.Chart(view).encode(x=alt.X("time:T", title=None))
|
|
|
|
| 373 |
)
|
| 374 |
st.altair_chart(band + q50 + obs)
|
| 375 |
st.caption("Shaded: q10–q90 band; solid: q50; dashed grey: truth.")
|
| 376 |
+
|
| 377 |
+
with tab_target:
|
| 378 |
+
tv = load_target()
|
| 379 |
+
st.markdown(
|
| 380 |
+
"Truth series from `v1/target.parquet`: mFRR activated **energy** (net, MWh per "
|
| 381 |
+
"quarter-hour, upward positive) and mFRR **price** (EUR/MWh), Spain, 15-min, "
|
| 382 |
+
"10 Dec 2024 – 31 Oct 2026. Missing energy is filled with 0 (no activation); missing "
|
| 383 |
+
"price stays empty. Times in Madrid local time.\n\n"
|
| 384 |
+
"- **Iberian blackout:** no data 28 Apr – 6 May 2025 (energy 0, price missing).\n"
|
| 385 |
+
"- **Regime shift:** prices rise from Jul 2026 (monthly mean ~100 → ~157 EUR/MWh through "
|
| 386 |
+
"Oct 2026), so recent months look different from the rest of the history."
|
| 387 |
+
)
|
| 388 |
+
target = st.radio("Target", ["energy", "price"], horizontal=True, key="tgt")
|
| 389 |
+
unit = "MWh" if target == "energy" else "EUR/MWh"
|
| 390 |
+
v = tv[tv.target == target]
|
| 391 |
+
|
| 392 |
+
st.subheader("Time series")
|
| 393 |
+
c1, c2 = st.columns([1, 2])
|
| 394 |
+
res = c1.radio("Resolution", ["Daily", "15-min"], horizontal=True)
|
| 395 |
+
if res == "Daily":
|
| 396 |
+
d = v.groupby("day").value.agg(["mean", "min", "max"]).reset_index()
|
| 397 |
+
base = alt.Chart(d).encode(x=alt.X("day:T", title=None))
|
| 398 |
+
chart = base.mark_area(opacity=0.2).encode(
|
| 399 |
+
y=alt.Y("min:Q", title=unit), y2="max:Q"
|
| 400 |
+
) + base.mark_line().encode(
|
| 401 |
+
y="mean:Q",
|
| 402 |
+
tooltip=[alt.Tooltip("day:T"), alt.Tooltip("mean:Q", format=".1f"), "min", "max"],
|
| 403 |
+
)
|
| 404 |
+
st.caption("Daily mean (line) and min–max (band).")
|
| 405 |
+
else:
|
| 406 |
+
lo, hi = v.day.min().date(), v.day.max().date()
|
| 407 |
+
rng = c2.date_input(
|
| 408 |
+
"Days (max 31)", value=(hi - dt.timedelta(days=13), hi), min_value=lo, max_value=hi
|
| 409 |
+
)
|
| 410 |
+
a, b = (rng[0], rng[-1]) if rng else (hi, hi)
|
| 411 |
+
b = min(b, a + dt.timedelta(days=30))
|
| 412 |
+
w = v[(v.day >= pd.Timestamp(a)) & (v.day <= pd.Timestamp(b))]
|
| 413 |
+
chart = (
|
| 414 |
+
alt.Chart(w)
|
| 415 |
+
.mark_line()
|
| 416 |
+
.encode(
|
| 417 |
+
x=alt.X("time:T", title=None),
|
| 418 |
+
y=alt.Y("value:Q", title=unit),
|
| 419 |
+
tooltip=["time:T", alt.Tooltip("value:Q", format=".1f")],
|
| 420 |
+
)
|
| 421 |
+
)
|
| 422 |
+
st.altair_chart(chart.interactive(bind_y=False))
|
| 423 |
+
|
| 424 |
+
st.subheader("Typical patterns")
|
| 425 |
+
st.caption("Mean (line) and q10–q90 (band) over the whole history.")
|
| 426 |
+
cols = st.columns(3)
|
| 427 |
+
cols[0].altair_chart(
|
| 428 |
+
band_chart(
|
| 429 |
+
profile(target, "slot"),
|
| 430 |
+
alt.X(
|
| 431 |
+
"slot:O", title=None, axis=alt.Axis(values=[f"{h:02d}:00" for h in range(0, 24, 3)])
|
| 432 |
+
),
|
| 433 |
+
"Quarter-hour of day",
|
| 434 |
+
)
|
| 435 |
+
)
|
| 436 |
+
wd = profile(target, "weekday").assign(
|
| 437 |
+
weekday=lambda d: d.weekday.map(dict(enumerate(WEEKDAYS)))
|
| 438 |
+
)
|
| 439 |
+
cols[1].altair_chart(band_chart(wd, alt.X("weekday:O", title=None, sort=WEEKDAYS), "Weekday"))
|
| 440 |
+
cols[2].altair_chart(
|
| 441 |
+
band_chart(profile(target, "month"), alt.X("yearmonth(month):T", title=None), "Month")
|
| 442 |
+
)
|
| 443 |
+
|
| 444 |
+
st.subheader("Distribution")
|
| 445 |
+
x = v.value.dropna()
|
| 446 |
+
stats = {
|
| 447 |
+
"Mean": f"{x.mean():.1f}",
|
| 448 |
+
"Std": f"{x.std():.1f}",
|
| 449 |
+
"Exact zeros": f"{(x == 0).mean():.1%}",
|
| 450 |
+
"Negative": f"{(x < 0).mean():.1%}",
|
| 451 |
+
"Min / max": f"{x.min():.0f} / {x.max():.0f}",
|
| 452 |
+
}
|
| 453 |
+
for col, (k, s) in zip(st.columns(len(stats)), stats.items(), strict=True):
|
| 454 |
+
col.metric(k, s)
|
| 455 |
+
lo, hi = x.quantile([0.005, 0.995])
|
| 456 |
+
counts, edges = np.histogram(x.clip(lo, hi), bins=80)
|
| 457 |
+
hist = pd.DataFrame({"start": edges[:-1], "end": edges[1:], "count": counts})
|
| 458 |
+
st.altair_chart(
|
| 459 |
+
alt.Chart(hist)
|
| 460 |
+
.mark_bar()
|
| 461 |
+
.encode(
|
| 462 |
+
x=alt.X("start:Q", bin="binned", title=unit),
|
| 463 |
+
x2="end:Q",
|
| 464 |
+
y=alt.Y("count:Q", title="Quarter-hours"),
|
| 465 |
+
tooltip=[
|
| 466 |
+
alt.Tooltip("start:Q", format=".1f"),
|
| 467 |
+
alt.Tooltip("end:Q", format=".1f"),
|
| 468 |
+
"count",
|
| 469 |
+
],
|
| 470 |
+
)
|
| 471 |
+
)
|
| 472 |
+
st.caption("Values clipped to the 0.5–99.5% range for display (extremes pile up at the edges).")
|
| 473 |
+
|
| 474 |
+
st.subheader("Missing values")
|
| 475 |
+
kind = st.radio("Count per day", ["Missing", "Exact zeros"], horizontal=True)
|
| 476 |
+
flag = v.value.isna() if kind == "Missing" else v.value.eq(0)
|
| 477 |
+
per_day = flag.groupby(v.day).sum().rename("n").reset_index()
|
| 478 |
+
per_day["month"] = per_day.day.dt.strftime("%Y-%m")
|
| 479 |
+
per_day["dom"] = per_day.day.dt.day
|
| 480 |
+
st.altair_chart(
|
| 481 |
+
alt.Chart(per_day)
|
| 482 |
+
.mark_rect()
|
| 483 |
+
.encode(
|
| 484 |
+
x=alt.X("dom:O", title="Day of month"),
|
| 485 |
+
y=alt.Y("month:O", title=None),
|
| 486 |
+
color=alt.Color("n:Q", scale=alt.Scale(scheme="oranges"), title="Quarter-hours"),
|
| 487 |
+
tooltip=[alt.Tooltip("day:T"), "n"],
|
| 488 |
+
)
|
| 489 |
+
)
|
| 490 |
+
st.caption(
|
| 491 |
+
f"{int(flag.sum())} of {len(flag)} quarter-hours ({flag.mean():.1%}). Energy gaps were "
|
| 492 |
+
"filled with 0 upstream, so for energy use 'Exact zeros' to see them."
|
| 493 |
+
)
|
| 494 |
+
|
| 495 |
+
st.subheader("Price vs energy")
|
| 496 |
+
both = pd.DataFrame(
|
| 497 |
+
{t: tv[tv.target == t].value.to_numpy() for t in ["energy", "price"]}
|
| 498 |
+
).dropna()
|
| 499 |
+
sample = both.sample(min(5000, len(both)), random_state=0)
|
| 500 |
+
st.altair_chart(
|
| 501 |
+
alt.Chart(sample)
|
| 502 |
+
.mark_circle(size=8, opacity=0.3)
|
| 503 |
+
.encode(
|
| 504 |
+
x=alt.X("energy:Q", title="Net mFRR energy (MWh)"),
|
| 505 |
+
y=alt.Y("price:Q", title="mFRR price (EUR/MWh)"),
|
| 506 |
+
tooltip=["energy", "price"],
|
| 507 |
+
)
|
| 508 |
+
)
|
| 509 |
+
st.caption("Random sample of 5000 quarter-hours.")
|