arnaudverstraeten commited on
Commit
9fcc2c2
·
verified ·
1 Parent(s): d878631

Leaderboard, win-rate matrix, target data tab

Browse files
Files changed (2) hide show
  1. README.md +2 -2
  2. app.py +358 -76
README.md CHANGED
@@ -6,9 +6,9 @@ colorTo: gray
6
  sdk: static
7
  pinned: false
8
  license: apache-2.0
9
- short_description: Backtest scores of open-predico forecasts for Predico
10
  ---
11
 
12
- Dashboard for [open-predico](https://github.com/arnaud-vs/open-predico). Reads backtest predictions from the public dataset [open-predico/benchmark](https://huggingface.co/datasets/open-predico/benchmark).
13
 
14
  Streamlit app run in the browser with [stlite](https://github.com/whitphx/stlite) (static Space, no server). Locally: `pip install -r requirements.txt && streamlit run app.py`.
 
6
  sdk: static
7
  pinned: false
8
  license: apache-2.0
9
+ short_description: Backtest leaderboard and target data for open-predico
10
  ---
11
 
12
+ Dashboard for [open-predico](https://github.com/arnaud-vs/open-predico). Backtest leaderboard, win rates and error breakdowns, plus an exploration of the target series. Reads the public dataset [open-predico/benchmark](https://huggingface.co/datasets/open-predico/benchmark).
13
 
14
  Streamlit app run in the browser with [stlite](https://github.com/whitphx/stlite) (static Space, no server). Locally: `pip install -r requirements.txt && streamlit run app.py`.
app.py CHANGED
@@ -1,9 +1,11 @@
1
- """open-predico dashboard: backtest scores from the public open-predico/benchmark dataset.
2
 
3
  Runs in the browser with stlite (Streamlit on Pyodide, see index.html), so the Space is a free
4
- static one; `streamlit run app.py` works locally too.
 
5
  """
6
 
 
7
  import io
8
  import json
9
  import sys
@@ -18,6 +20,20 @@ HF = "https://huggingface.co"
18
  PREFIX = "predictions/backtest/"
19
  TZ = "Europe/Madrid"
20
  ALPHA = 0.2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
 
22
  COLOR = alt.Color(
23
  "config:N", scale=alt.Scale(scheme="tableau10"), legend=alt.Legend(orient="bottom")
@@ -44,9 +60,20 @@ def fetch(url: str) -> bytes:
44
  return req.response.to_py().tobytes()
45
 
46
 
47
- @st.cache_data(ttl=3600)
48
- def load() -> pd.DataFrame:
49
- """Per-step errors for every config and target, with local time columns."""
 
 
 
 
 
 
 
 
 
 
 
50
  tree = json.loads(
51
  fetch(f"{HF}/api/datasets/{REPO}/tree/main/{PREFIX.rstrip('/')}?recursive=true")
52
  )
@@ -54,7 +81,7 @@ def load() -> pd.DataFrame:
54
  out = []
55
  for f in files:
56
  config, target = f.removeprefix(PREFIX).removesuffix(".parquet").split("/")
57
- df = pd.read_parquet(io.BytesIO(fetch(f"{HF}/datasets/{REPO}/resolve/main/{f}")))
58
  # Same formulas as src/openpredico/evaluation/metrics.py (copied, keep in sync).
59
  o, q10, q50, q90 = df.obs, df.q10, df.q50, df.q90
60
  df["se"] = (o - q50) ** 2
@@ -70,14 +97,15 @@ def load() -> pd.DataFrame:
70
  df["day"] = local.tz_localize(None).normalize()
71
  df["hour"] = local.hour
72
  df["weekday"] = local.dayofweek
73
- df["config"], df["target"] = config, target
74
  out.append(df.reset_index())
75
  return pd.concat(out, ignore_index=True)
76
 
77
 
78
- def daily(steps: pd.DataFrame) -> pd.DataFrame:
 
79
  """Daily scores; a day with any missing truth is skipped, as Predico does."""
80
- g = steps.groupby(["target", "config", "day"])
81
  d = g.agg(
82
  se=("se", "mean"),
83
  winkler=("winkler", "mean"),
@@ -88,10 +116,91 @@ def daily(steps: pd.DataFrame) -> pd.DataFrame:
88
  return d[~d.pop("missing")].reset_index()
89
 
90
 
91
- steps = load()
92
- days = daily(steps)
93
- targets = sorted(days.target.unique())
94
- configs = sorted(days.config.unique())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
 
96
  st.title("open-predico")
97
  st.caption(
@@ -99,97 +208,135 @@ st.caption(
99
  "[Code](https://github.com/arnaud-vs/open-predico) · "
100
  "[Data](https://huggingface.co/datasets/open-predico/benchmark)"
101
  )
102
- (tab,) = st.tabs(["Backtest"])
103
 
104
- with tab:
 
 
105
  st.markdown(
106
- "Test year 1 Nov 2025 – 31 Oct 2026, forecast made at 14:00 Madrid on D-1. "
107
- "RMSE on q50, Winkler (α = 0.2) on q10/q90, per local day; lower is better. "
108
- "Days with any missing truth are skipped, like Predico."
109
  )
110
 
111
  st.subheader("Leaderboard")
112
- cols = st.columns(len(targets))
113
- for col, t in zip(cols, targets, strict=True):
114
- lb = (
115
- days[days.target == t]
116
- .groupby("config")
117
- .agg(
118
- rmse=("rmse", "mean"),
119
- winkler=("winkler", "mean"),
120
- coverage=("coverage", "mean"),
121
- days=("day", "count"),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  )
123
- .sort_values("rmse")
124
  )
125
- col.markdown(f"**{t}**")
126
- col.dataframe(
127
- lb.style.format({"rmse": "{:.1f}", "winkler": "{:.1f}", "coverage": "{:.0%}"}),
 
128
  )
 
129
 
130
  st.subheader("By month")
131
- target = st.selectbox("Target", targets, key="month_target")
132
  monthly = (
133
  days[days.target == target]
134
  .assign(month=lambda d: d.day.dt.to_period("M").dt.to_timestamp())
135
- .groupby(["config", "month"])[["rmse", "winkler"]]
136
  .mean()
137
  .reset_index()
138
  )
139
  cols = st.columns(2)
140
- for col, metric in zip(cols, ["rmse", "winkler"], strict=True):
141
  col.altair_chart(
142
- alt.Chart(monthly, title=metric.upper())
143
  .mark_line(point=True)
144
  .encode(
145
  x=alt.X("yearmonth(month):T", title=None),
146
- y=alt.Y(f"{metric}:Q", title=None),
147
  color=COLOR,
148
- tooltip=["config", "yearmonth(month):T", alt.Tooltip(f"{metric}:Q", format=".1f")],
149
  ),
150
  )
151
 
152
- st.subheader("Paired comparison")
153
- c1, c2 = st.columns(2)
154
- a = c1.selectbox("Config A", configs, index=len(configs) - 1)
155
- b = c2.selectbox("Config B", configs, index=0)
156
- rows = []
157
- for t in targets:
158
- p = days[days.target == t].pivot(index="day", columns="config", values=["rmse", "winkler"])
159
- both = p.dropna()
160
- rows.append(
161
- {
162
- "target": t,
163
- "days": len(both),
164
- "A wins RMSE": (both["rmse"][a] < both["rmse"][b]).mean(),
165
- "A wins Winkler": (both["winkler"][a] < both["winkler"][b]).mean(),
166
- }
167
- )
168
- st.dataframe(
169
- pd.DataFrame(rows)
170
- .set_index("target")
171
- .style.format({"A wins RMSE": "{:.0%}", "A wins Winkler": "{:.0%}"}),
172
- )
173
-
174
  st.subheader("Absolute error of q50 by hour and weekday (Madrid time)")
175
- target = st.selectbox("Target", targets, key="hour_target")
176
  sel = steps[steps.target == target]
177
  cols = st.columns(2)
178
  for col, key, title in [(cols[0], "hour", "Hour of day"), (cols[1], "weekday", "Weekday")]:
179
  prof = sel.groupby(["config", key]).ae.mean().reset_index()
180
  if key == "weekday":
181
- prof["weekday"] = prof.weekday.map(
182
- dict(enumerate("Mon Tue Wed Thu Fri Sat Sun".split()))
183
- )
184
  col.altair_chart(
185
  alt.Chart(prof, title=title)
186
  .mark_line(point=True)
187
  .encode(
188
- x=alt.X(
189
- f"{key}:O",
190
- title=None,
191
- sort="Mon Tue Wed Thu Fri Sat Sun".split() if key == "weekday" else None,
192
- ),
193
  y=alt.Y("ae:Q", title="MAE"),
194
  color=COLOR,
195
  tooltip=["config", key, alt.Tooltip("ae:Q", format=".1f")],
@@ -198,21 +345,22 @@ with tab:
198
 
199
  st.subheader("Day viewer")
200
  c1, c2 = st.columns(2)
201
- target = c1.selectbox("Target", targets, key="day_target")
202
  lo, hi = steps.day.min().date(), steps.day.max().date()
203
  date = c2.date_input("Day (Madrid)", value=hi, min_value=lo, max_value=hi)
204
- view = steps[(steps.target == target) & (steps.day == pd.Timestamp(date))].copy()
 
 
 
205
  view["time"] = view.time.dt.tz_convert(TZ).dt.tz_localize(None)
206
- cfgs = st.multiselect("Configs", configs, default=configs)
207
- view = view[view.config.isin(cfgs)]
208
- shown = days[(days.target == target) & (days.day == pd.Timestamp(date))]
209
- if shown.empty:
210
  st.caption("Truth incomplete on this day: not scored.")
211
  else:
212
  st.caption(
213
  " · ".join(
214
  f"{r.config}: RMSE {r.rmse:.1f}, Winkler {r.winkler:.1f}"
215
- for r in shown.itertuples()
216
  )
217
  )
218
  base = alt.Chart(view).encode(x=alt.X("time:T", title=None))
@@ -225,3 +373,137 @@ with tab:
225
  )
226
  st.altair_chart(band + q50 + obs)
227
  st.caption("Shaded: q10–q90 band; solid: q50; dashed grey: truth.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """open-predico dashboard: backtest scores and target data from open-predico/benchmark (public).
2
 
3
  Runs in the browser with stlite (Streamlit on Pyodide, see index.html), so the Space is a free
4
+ static one; `streamlit run app.py` works locally too. Pyodide is slow: every file is fetched once
5
+ and cached, and plots get aggregated data.
6
  """
7
 
8
+ import datetime as dt
9
  import io
10
  import json
11
  import sys
 
20
  PREFIX = "predictions/backtest/"
21
  TZ = "Europe/Madrid"
22
  ALPHA = 0.2
23
+ BASELINE = "slot_climatology_28d"
24
+ METRICS = {"rmse": "RMSE", "winkler": "Winkler"}
25
+ WEEKDAYS = "Mon Tue Wed Thu Fri Sat Sun".split()
26
+ # Model type from the config name prefix; anything else is an ML model.
27
+ TYPES = {
28
+ "slot_climatology": "baseline",
29
+ "persistence": "baseline",
30
+ "zero_inflated": "baseline",
31
+ "chronos": "foundation model",
32
+ "timesfm": "foundation model",
33
+ "timestral": "foundation model",
34
+ "t0": "foundation model",
35
+ "toto": "foundation model",
36
+ }
37
 
38
  COLOR = alt.Color(
39
  "config:N", scale=alt.Scale(scheme="tableau10"), legend=alt.Legend(orient="bottom")
 
60
  return req.response.to_py().tobytes()
61
 
62
 
63
+ def parquet(path: str) -> pd.DataFrame:
64
+ return pd.read_parquet(io.BytesIO(fetch(f"{HF}/datasets/{REPO}/resolve/main/{path}")))
65
+
66
+
67
+ def model_type(config: str) -> str:
68
+ return next((t for p, t in TYPES.items() if config.startswith(p)), "ML")
69
+
70
+
71
+ # --- Backtest data ---------------------------------------------------------------------------
72
+
73
+
74
+ @st.cache_resource(ttl=3600)
75
+ def load_steps() -> pd.DataFrame:
76
+ """Per-step errors for every config folder under predictions/backtest/, with local time."""
77
  tree = json.loads(
78
  fetch(f"{HF}/api/datasets/{REPO}/tree/main/{PREFIX.rstrip('/')}?recursive=true")
79
  )
 
81
  out = []
82
  for f in files:
83
  config, target = f.removeprefix(PREFIX).removesuffix(".parquet").split("/")
84
+ df = parquet(f)
85
  # Same formulas as src/openpredico/evaluation/metrics.py (copied, keep in sync).
86
  o, q10, q50, q90 = df.obs, df.q10, df.q50, df.q90
87
  df["se"] = (o - q50) ** 2
 
97
  df["day"] = local.tz_localize(None).normalize()
98
  df["hour"] = local.hour
99
  df["weekday"] = local.dayofweek
100
+ df["config"], df["target"] = config, target.removeprefix("spain_mfrr_")
101
  out.append(df.reset_index())
102
  return pd.concat(out, ignore_index=True)
103
 
104
 
105
+ @st.cache_resource(ttl=3600)
106
+ def load_days() -> pd.DataFrame:
107
  """Daily scores; a day with any missing truth is skipped, as Predico does."""
108
+ g = load_steps().groupby(["target", "config", "day"])
109
  d = g.agg(
110
  se=("se", "mean"),
111
  winkler=("winkler", "mean"),
 
116
  return d[~d.pop("missing")].reset_index()
117
 
118
 
119
+ def leaderboard(days: pd.DataFrame) -> pd.DataFrame:
120
+ """One row per config: raw and baseline-normalised scores per target, aggregate and rank.
121
+
122
+ Normalised = mean daily score / baseline's mean daily score, on days both are scored.
123
+ Score = geometric mean of the normalised RMSE and Winkler over both targets.
124
+ """
125
+ cols = {}
126
+ for t, d in days.groupby("target"):
127
+ p = d.pivot(index="day", columns="config", values=["rmse", "winkler", "coverage"])
128
+ for m in METRICS:
129
+ cols[f"{t} {m}"] = p[m].mean()
130
+ if BASELINE in p[m]:
131
+ both = p[m].dropna(subset=[BASELINE])
132
+ cols[f"{t} {m} rel"] = both.mean() / both[BASELINE].mean()
133
+ cols[f"{t} coverage"] = p["coverage"].mean()
134
+ cols[f"{t} days"] = p["rmse"].count()
135
+ lb = pd.DataFrame(cols)
136
+ rel = lb.filter(like=" rel")
137
+ lb.insert(0, "score", np.exp(np.log(rel).mean(axis=1, skipna=False)))
138
+ lb.insert(0, "type", [model_type(c) for c in lb.index])
139
+ lb.insert(0, "rank", lb.score.rank(method="min"))
140
+ return lb.sort_values(["score", "rank"]).rename_axis("config").reset_index()
141
+
142
+
143
+ def win_rates(days: pd.DataFrame, target: str, metric: str, order: list[str]) -> pd.DataFrame:
144
+ """Long frame: share of commonly scored days where row config A beats column config B."""
145
+ p = days[days.target == target].pivot(index="day", columns="config", values=metric)
146
+ rows = []
147
+ for a in order:
148
+ for b in order:
149
+ both = p[[a, b]].dropna() if a != b else None
150
+ rows.append(
151
+ {
152
+ "A": a,
153
+ "B": b,
154
+ "win": (both[a] < both[b]).mean() if a != b and len(both) else np.nan,
155
+ "days": len(both) if a != b else 0,
156
+ }
157
+ )
158
+ return pd.DataFrame(rows)
159
+
160
+
161
+ # --- Target data -----------------------------------------------------------------------------
162
+
163
+
164
+ @st.cache_resource
165
+ def load_target() -> pd.DataFrame:
166
+ """v1/target.parquet in long form with local calendar columns."""
167
+ df = parquet("v1/target.parquet")
168
+ local = df.index.tz_convert(TZ)
169
+ cal = pd.DataFrame(
170
+ {
171
+ "time": local.tz_localize(None),
172
+ "day": local.tz_localize(None).normalize(),
173
+ "slot": local.strftime("%H:%M"),
174
+ "weekday": local.dayofweek,
175
+ "month": local.tz_localize(None).to_period("M").to_timestamp(),
176
+ }
177
+ )
178
+ out = []
179
+ for col in df.columns:
180
+ out.append(cal.assign(target=col.removeprefix("mfrr_"), value=df[col].to_numpy()))
181
+ return pd.concat(out, ignore_index=True)
182
+
183
+
184
+ @st.cache_resource
185
+ def profile(target: str, key: str) -> pd.DataFrame:
186
+ """Mean and q10/q90 of the target by a calendar key."""
187
+ v = load_target()
188
+ g = v[v.target == target].groupby(key).value
189
+ return pd.DataFrame(
190
+ {"mean": g.mean(), "q10": g.quantile(0.1), "q90": g.quantile(0.9)}
191
+ ).reset_index()
192
+
193
+
194
+ def band_chart(df: pd.DataFrame, x: alt.X, title: str) -> alt.LayerChart:
195
+ base = alt.Chart(df, title=title).encode(x=x)
196
+ band = base.mark_area(opacity=0.25).encode(y=alt.Y("q10:Q", title=None), y2="q90:Q")
197
+ line = base.mark_line(point=len(df) < 40).encode(
198
+ y="mean:Q", tooltip=[x.shorthand, alt.Tooltip("mean:Q", format=".1f")]
199
+ )
200
+ return band + line
201
+
202
+
203
+ # --- Page ------------------------------------------------------------------------------------
204
 
205
  st.title("open-predico")
206
  st.caption(
 
208
  "[Code](https://github.com/arnaud-vs/open-predico) · "
209
  "[Data](https://huggingface.co/datasets/open-predico/benchmark)"
210
  )
211
+ tab_bt, tab_target = st.tabs(["Backtest", "Target data"])
212
 
213
+ with tab_bt:
214
+ steps, days = load_steps(), load_days()
215
+ targets = sorted(days.target.unique())
216
  st.markdown(
217
+ "Test year 1 Nov 2025 – 31 Oct 2026, forecast made at 14:00 Madrid on D-1, scored per "
218
+ "local day; lower is better. Days with any missing truth are skipped, like Predico."
 
219
  )
220
 
221
  st.subheader("Leaderboard")
222
+ lb = leaderboard(days)
223
+ order = lb.config.tolist()
224
+ types = st.pills("Model type", sorted(lb.type.unique()), selection_mode="multi", default=None)
225
+ absolute = st.toggle("Show absolute scores")
226
+ shown = lb[lb.type.isin(types)] if types else lb
227
+ keep = ["rank", "config", "type", "score"]
228
+ for t in targets:
229
+ keep += [f"{t} rmse rel", f"{t} winkler rel", f"{t} coverage"]
230
+ keep += [f"{t} rmse", f"{t} winkler", f"{t} days"] if absolute else []
231
+ ratio = "Mean daily {} / baseline's, on days both are scored; < 1 beats the baseline."
232
+ colcfg = {
233
+ "rank": st.column_config.NumberColumn("Rank", format="%d"),
234
+ "score": st.column_config.NumberColumn(
235
+ "Score",
236
+ format="%.3f",
237
+ help="Geometric mean of the four normalised scores (RMSE and Winkler, both targets).",
238
+ ),
239
+ }
240
+ for t in targets:
241
+ colcfg |= {
242
+ f"{t} rmse rel": st.column_config.NumberColumn(
243
+ f"{t} RMSE", format="%.3f", help=ratio.format("RMSE of q50")
244
+ ),
245
+ f"{t} winkler rel": st.column_config.NumberColumn(
246
+ f"{t} Winkler", format="%.3f", help=ratio.format("Winkler score of q10/q90")
247
+ ),
248
+ f"{t} coverage": st.column_config.NumberColumn(
249
+ f"{t} coverage",
250
+ format="percent",
251
+ help="Share of quarter-hours inside q10–q90; ideal 80%.",
252
+ ),
253
+ f"{t} rmse": st.column_config.NumberColumn(f"{t} RMSE abs", format="%.1f"),
254
+ f"{t} winkler": st.column_config.NumberColumn(f"{t} Winkler abs", format="%.1f"),
255
+ f"{t} days": st.column_config.NumberColumn(f"{t} days", format="%d"),
256
+ }
257
+ st.dataframe(shown[keep], column_config=colcfg, hide_index=True)
258
+ with st.expander("Metrics"):
259
+ st.markdown(
260
+ f"""
261
+ - **RMSE** (Predico's q50 score): `sqrt(mean_t (obs − q50)²)` over a local day.
262
+ - **Winkler** (Predico's q10/q90 score, α = 0.2): `mean_t [(q90 − q10) + 10·max(0, q10 − obs)
263
+ + 10·max(0, obs − q90)]`. Rewards narrow intervals, penalises misses.
264
+ - **Coverage**: share of quarter-hours with obs inside [q10, q90]; a calibrated model gets 80%.
265
+ - **Normalised**: a config's mean daily score divided by the baseline `{BASELINE}` (= 1.0) on
266
+ the days both are scored. **Score** is the geometric mean of the four normalised values;
267
+ **Rank** sorts by it.
268
+ - **Type**: baseline, foundation model (pretrained, zero-shot), ML (trained here).
269
+ """
270
+ )
271
+
272
+ st.subheader("Win rate")
273
+ st.caption(
274
+ "Cell (row A, column B): share of days scored for both where A has the lower score. "
275
+ "Read across a row: green = A usually wins."
276
+ )
277
+ target = st.radio("Target", targets, horizontal=True, key="win_target")
278
+ cols = st.columns(2)
279
+ for col, (m, label) in zip(cols, METRICS.items(), strict=True):
280
+ w = win_rates(days, target, m, order)
281
+ enc = {
282
+ "x": alt.X("B:N", sort=order, title="B"),
283
+ "y": alt.Y("A:N", sort=order, title="A"),
284
+ }
285
+ heat = (
286
+ alt.Chart(w, title=label)
287
+ .mark_rect()
288
+ .encode(
289
+ **enc,
290
+ color=alt.Color(
291
+ "win:Q",
292
+ scale=alt.Scale(scheme="redyellowgreen", domain=[0, 1]),
293
+ legend=alt.Legend(format="%", title="A wins"),
294
+ ),
295
+ tooltip=["A", "B", alt.Tooltip("win:Q", format=".0%"), "days"],
296
  )
 
297
  )
298
+ text = (
299
+ alt.Chart(w)
300
+ .mark_text(color="black")
301
+ .encode(**enc, text=alt.Text("win:Q", format=".0%"))
302
  )
303
+ col.altair_chart(heat + text)
304
 
305
  st.subheader("By month")
306
+ target = st.radio("Target", targets, horizontal=True, key="month_target")
307
  monthly = (
308
  days[days.target == target]
309
  .assign(month=lambda d: d.day.dt.to_period("M").dt.to_timestamp())
310
+ .groupby(["config", "month"])[list(METRICS)]
311
  .mean()
312
  .reset_index()
313
  )
314
  cols = st.columns(2)
315
+ for col, (m, label) in zip(cols, METRICS.items(), strict=True):
316
  col.altair_chart(
317
+ alt.Chart(monthly, title=label)
318
  .mark_line(point=True)
319
  .encode(
320
  x=alt.X("yearmonth(month):T", title=None),
321
+ y=alt.Y(f"{m}:Q", title=None),
322
  color=COLOR,
323
+ tooltip=["config", "yearmonth(month):T", alt.Tooltip(f"{m}:Q", format=".1f")],
324
  ),
325
  )
326
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
  st.subheader("Absolute error of q50 by hour and weekday (Madrid time)")
328
+ target = st.radio("Target", targets, horizontal=True, key="hour_target")
329
  sel = steps[steps.target == target]
330
  cols = st.columns(2)
331
  for col, key, title in [(cols[0], "hour", "Hour of day"), (cols[1], "weekday", "Weekday")]:
332
  prof = sel.groupby(["config", key]).ae.mean().reset_index()
333
  if key == "weekday":
334
+ prof["weekday"] = prof.weekday.map(dict(enumerate(WEEKDAYS)))
 
 
335
  col.altair_chart(
336
  alt.Chart(prof, title=title)
337
  .mark_line(point=True)
338
  .encode(
339
+ x=alt.X(f"{key}:O", title=None, sort=WEEKDAYS if key == "weekday" else None),
 
 
 
 
340
  y=alt.Y("ae:Q", title="MAE"),
341
  color=COLOR,
342
  tooltip=["config", key, alt.Tooltip("ae:Q", format=".1f")],
 
345
 
346
  st.subheader("Day viewer")
347
  c1, c2 = st.columns(2)
348
+ target = c1.radio("Target", targets, horizontal=True, key="day_target")
349
  lo, hi = steps.day.min().date(), steps.day.max().date()
350
  date = c2.date_input("Day (Madrid)", value=hi, min_value=lo, max_value=hi)
351
+ cfgs = st.multiselect("Configs", order, default=order)
352
+ view = steps[
353
+ (steps.target == target) & (steps.day == pd.Timestamp(date)) & steps.config.isin(cfgs)
354
+ ].copy()
355
  view["time"] = view.time.dt.tz_convert(TZ).dt.tz_localize(None)
356
+ scored = days[(days.target == target) & (days.day == pd.Timestamp(date))]
357
+ if scored.empty:
 
 
358
  st.caption("Truth incomplete on this day: not scored.")
359
  else:
360
  st.caption(
361
  " · ".join(
362
  f"{r.config}: RMSE {r.rmse:.1f}, Winkler {r.winkler:.1f}"
363
+ for r in scored.itertuples()
364
  )
365
  )
366
  base = alt.Chart(view).encode(x=alt.X("time:T", title=None))
 
373
  )
374
  st.altair_chart(band + q50 + obs)
375
  st.caption("Shaded: q10–q90 band; solid: q50; dashed grey: truth.")
376
+
377
+ with tab_target:
378
+ tv = load_target()
379
+ st.markdown(
380
+ "Truth series from `v1/target.parquet`: mFRR activated **energy** (net, MWh per "
381
+ "quarter-hour, upward positive) and mFRR **price** (EUR/MWh), Spain, 15-min, "
382
+ "10 Dec 2024 – 31 Oct 2026. Missing energy is filled with 0 (no activation); missing "
383
+ "price stays empty. Times in Madrid local time.\n\n"
384
+ "- **Iberian blackout:** no data 28 Apr – 6 May 2025 (energy 0, price missing).\n"
385
+ "- **Regime shift:** prices rise from Jul 2026 (monthly mean ~100 → ~157 EUR/MWh through "
386
+ "Oct 2026), so recent months look different from the rest of the history."
387
+ )
388
+ target = st.radio("Target", ["energy", "price"], horizontal=True, key="tgt")
389
+ unit = "MWh" if target == "energy" else "EUR/MWh"
390
+ v = tv[tv.target == target]
391
+
392
+ st.subheader("Time series")
393
+ c1, c2 = st.columns([1, 2])
394
+ res = c1.radio("Resolution", ["Daily", "15-min"], horizontal=True)
395
+ if res == "Daily":
396
+ d = v.groupby("day").value.agg(["mean", "min", "max"]).reset_index()
397
+ base = alt.Chart(d).encode(x=alt.X("day:T", title=None))
398
+ chart = base.mark_area(opacity=0.2).encode(
399
+ y=alt.Y("min:Q", title=unit), y2="max:Q"
400
+ ) + base.mark_line().encode(
401
+ y="mean:Q",
402
+ tooltip=[alt.Tooltip("day:T"), alt.Tooltip("mean:Q", format=".1f"), "min", "max"],
403
+ )
404
+ st.caption("Daily mean (line) and min–max (band).")
405
+ else:
406
+ lo, hi = v.day.min().date(), v.day.max().date()
407
+ rng = c2.date_input(
408
+ "Days (max 31)", value=(hi - dt.timedelta(days=13), hi), min_value=lo, max_value=hi
409
+ )
410
+ a, b = (rng[0], rng[-1]) if rng else (hi, hi)
411
+ b = min(b, a + dt.timedelta(days=30))
412
+ w = v[(v.day >= pd.Timestamp(a)) & (v.day <= pd.Timestamp(b))]
413
+ chart = (
414
+ alt.Chart(w)
415
+ .mark_line()
416
+ .encode(
417
+ x=alt.X("time:T", title=None),
418
+ y=alt.Y("value:Q", title=unit),
419
+ tooltip=["time:T", alt.Tooltip("value:Q", format=".1f")],
420
+ )
421
+ )
422
+ st.altair_chart(chart.interactive(bind_y=False))
423
+
424
+ st.subheader("Typical patterns")
425
+ st.caption("Mean (line) and q10–q90 (band) over the whole history.")
426
+ cols = st.columns(3)
427
+ cols[0].altair_chart(
428
+ band_chart(
429
+ profile(target, "slot"),
430
+ alt.X(
431
+ "slot:O", title=None, axis=alt.Axis(values=[f"{h:02d}:00" for h in range(0, 24, 3)])
432
+ ),
433
+ "Quarter-hour of day",
434
+ )
435
+ )
436
+ wd = profile(target, "weekday").assign(
437
+ weekday=lambda d: d.weekday.map(dict(enumerate(WEEKDAYS)))
438
+ )
439
+ cols[1].altair_chart(band_chart(wd, alt.X("weekday:O", title=None, sort=WEEKDAYS), "Weekday"))
440
+ cols[2].altair_chart(
441
+ band_chart(profile(target, "month"), alt.X("yearmonth(month):T", title=None), "Month")
442
+ )
443
+
444
+ st.subheader("Distribution")
445
+ x = v.value.dropna()
446
+ stats = {
447
+ "Mean": f"{x.mean():.1f}",
448
+ "Std": f"{x.std():.1f}",
449
+ "Exact zeros": f"{(x == 0).mean():.1%}",
450
+ "Negative": f"{(x < 0).mean():.1%}",
451
+ "Min / max": f"{x.min():.0f} / {x.max():.0f}",
452
+ }
453
+ for col, (k, s) in zip(st.columns(len(stats)), stats.items(), strict=True):
454
+ col.metric(k, s)
455
+ lo, hi = x.quantile([0.005, 0.995])
456
+ counts, edges = np.histogram(x.clip(lo, hi), bins=80)
457
+ hist = pd.DataFrame({"start": edges[:-1], "end": edges[1:], "count": counts})
458
+ st.altair_chart(
459
+ alt.Chart(hist)
460
+ .mark_bar()
461
+ .encode(
462
+ x=alt.X("start:Q", bin="binned", title=unit),
463
+ x2="end:Q",
464
+ y=alt.Y("count:Q", title="Quarter-hours"),
465
+ tooltip=[
466
+ alt.Tooltip("start:Q", format=".1f"),
467
+ alt.Tooltip("end:Q", format=".1f"),
468
+ "count",
469
+ ],
470
+ )
471
+ )
472
+ st.caption("Values clipped to the 0.5–99.5% range for display (extremes pile up at the edges).")
473
+
474
+ st.subheader("Missing values")
475
+ kind = st.radio("Count per day", ["Missing", "Exact zeros"], horizontal=True)
476
+ flag = v.value.isna() if kind == "Missing" else v.value.eq(0)
477
+ per_day = flag.groupby(v.day).sum().rename("n").reset_index()
478
+ per_day["month"] = per_day.day.dt.strftime("%Y-%m")
479
+ per_day["dom"] = per_day.day.dt.day
480
+ st.altair_chart(
481
+ alt.Chart(per_day)
482
+ .mark_rect()
483
+ .encode(
484
+ x=alt.X("dom:O", title="Day of month"),
485
+ y=alt.Y("month:O", title=None),
486
+ color=alt.Color("n:Q", scale=alt.Scale(scheme="oranges"), title="Quarter-hours"),
487
+ tooltip=[alt.Tooltip("day:T"), "n"],
488
+ )
489
+ )
490
+ st.caption(
491
+ f"{int(flag.sum())} of {len(flag)} quarter-hours ({flag.mean():.1%}). Energy gaps were "
492
+ "filled with 0 upstream, so for energy use 'Exact zeros' to see them."
493
+ )
494
+
495
+ st.subheader("Price vs energy")
496
+ both = pd.DataFrame(
497
+ {t: tv[tv.target == t].value.to_numpy() for t in ["energy", "price"]}
498
+ ).dropna()
499
+ sample = both.sample(min(5000, len(both)), random_state=0)
500
+ st.altair_chart(
501
+ alt.Chart(sample)
502
+ .mark_circle(size=8, opacity=0.3)
503
+ .encode(
504
+ x=alt.X("energy:Q", title="Net mFRR energy (MWh)"),
505
+ y=alt.Y("price:Q", title="mFRR price (EUR/MWh)"),
506
+ tooltip=["energy", "price"],
507
+ )
508
+ )
509
+ st.caption("Random sample of 5000 quarter-hours.")