Time Series Forecasting
PyTorch
English
pluvium
weather
weather-forecasting
climate
meteorology
earth-science
precipitation
rainfall
extreme-rainfall
nowcasting
probabilistic-forecasting
tropical-cyclones
hurricane
multi-model-ensemble
post-processing
online-learning
geospatial
caribbean
naturev1
ihelix
graphcast
aifs
File size: 29,108 Bytes
fbb9a05 527d56f fbb9a05 527d56f fbb9a05 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 | # Copyright 2026 Nathan. Apache-2.0.
"""Write docs/FRONTIER_BENCHMARK.md from the benchmark's own output, so every number in it is the data's.
python scripts/doc_frontier.py RESULTS FIRST EXPERIMENTS docs/FRONTIER_BENCHMARK.md [docs/GUIDE.md docs/MODEL_CARD.md ...]
``RESULTS`` holds, per place, ``{place}.json`` (from :func:`pluvium.frontier.evaluate.evaluate`), ``{place}_tail.json``,
``{place}_scenarios.json`` and, for the US places, ``{place}_winter_DJF.json``. ``FIRST`` holds the first benchmark's
``{place}.json``. ``EXPERIMENTS`` holds the withheld-storm results ``{place}.json`` (optional). Any further pages
named get the headline results written between their ``<!-- frontier-summary:start -->`` and ``:end`` markers.
"""
import sys
from pathlib import Path
import numpy as np
from pluvium.benchmarks import fullyear as F
from pluvium.benchmarks import scenario_bench, tail_bench, winter_bench
from pluvium.frontier import report as R
results_dir, first_dir, experiments_dir, out = (Path(p) for p in sys.argv[1:5])
ORDER, SHORT = R.PLACE_ORDER, R.PLACE_NAMES
WINTER = {"alps": "DJF", "pacific_nw": "DJF", "great_plains": "DJF", "andes_south": "JJA"}
#: the places scored before the rule that decides the hand-over was final, and the places scored after
SEEN, AFTER = R.SEEN_FIRST, R.SCORED_AFTER
load = R.load_results
def names(places) -> str:
return R.join_names(places, the=True)
new, old = load(results_dir), load(first_dir)
tails, scenarios = load(results_dir, "_tail"), load(results_dir, "_scenarios")
winters = load(results_dir, "_winter_DJF")
withheld = load(experiments_dir) if experiments_dir.exists() else {}
late_dir = results_dir.parent / "results_late140" # the test year with the truth 140 days late
late = load(late_dir) if late_dir.exists() else {}
late = {k: v for k, v in late.items() if k in new}
old = {k: v for k, v in old.items() if k in new}
G = list(F.LEAD_GROUPS)
seen = [p for p in ORDER if p in new and p in SEEN]
after = [p for p in ORDER if p in new and p in AFTER]
def count(key, t, higher=True, groups=None, season=None, places=None):
use_new = {k: v for k, v in new.items() if places is None or k in places}
return R.tally(old, use_new, key, t, higher, groups, season)
def value(place, model, key, t, g):
return F._value(new[place]["groups"][g].get(model), key, t)
def mean_of(results, model, key, t, g, season=None):
vals = []
for res in results.values():
row = res["groups"][g] if season is None else res["seasons"][g][season]
name = F._top(res) if model == "top" else model
v = F._value(row.get(name), key, t)
if v == v:
vals.append(v)
return float(np.mean(vals)) if vals else float("nan")
def confident(threshold: str, groups) -> tuple:
"""(comparisons, better with 95% confidence, worse with 95% confidence) of the Blend against every post-processed
system at a threshold (bootstrap over runs), over places and lead groups."""
n = better = worse = 0
for res in new.values():
for g in groups:
for rival, v in res["versus"].get(g, {}).get(threshold, {}).items():
if rival in R.ABLATIONS or rival in ("blend_ops", "neural_pp", "blend"):
continue
n += 1
better += v["ci95"][1] < 0
worse += v["ci95"][0] > 0
return n, better, worse
def board_rows() -> list:
lines = ["| Score | " + " | ".join(G) + " |", "|---|" + "---|" * len(G)]
for title, key, t, higher in F.BOARD:
cells = []
for g in G:
c = count(key, t, higher, [g])
cells.append(f"{c['new_leads']} of {c['cells']} (was {c['old_leads']})" if c["cells"] else "-")
lines.append(f"| {title} | " + " | ".join(cells) + " |")
return lines
def winter_rows() -> list:
lines = ["| Place (its winter) | " + " | ".join(G) + " |", "|---|" + "---|" * len(G)]
for place, season in WINTER.items():
if place not in new:
continue
cells = []
for g in G:
row = new[place]["seasons"][g][season]
b = F._value(row.get("stack"), "bss", "1")
a = F._value(old[place]["seasons"][g][season].get(F._top(old[place])), "bss", "1") if place in old else float("nan")
rival, who = R.best_rival(row, "bss", "1", True)
if b != b:
cells.append("-")
continue
mark = "**" if rival != rival or b > rival else ""
cells.append(f"{a:+.3f} -> {mark}{b:+.3f}{mark} ({rival:+.3f}, {F.LABELS.get(str(who).replace('_raw', ''), who)})")
lines.append(f"| {SHORT[place]} ({season}) | " + " | ".join(cells) + " |")
return lines
def melissa_rows() -> list:
"""Jamaica on the landfall day: the first Blend, the new one, the raw consensus."""
if "jamaica" not in tails:
return []
day = "2025-10-28"
rows = sorted((c for c in tails["jamaica"]["cases"] if c["valid"] == day), key=lambda c: c["lead"])
if not rows:
return []
first = {c["lead"]: c for c in old.get("jamaica", {}).get("cases", []) if c["valid"] == day and c["model"] == F._top(old["jamaica"])}
o = rows[0]
lines = [f"Observed over Jamaica on {day} (CMORPH): {o['observed_mean_mm']:.0f} mm on average, up to "
f"{o['observed_max_mm']:.0f} mm; {100 * o['observed_share_50']:.0f}% of places had 50 mm or more and "
f"{100 * o['observed_share_100']:.0f}% had 100 mm or more.", "",
"| Days ahead | First Blend: chance of 50 mm, expected mm | New Blend: chance of 50 mm, of 100 mm, expected mm | "
"Places under an extreme warning | Raw consensus: chance of 50 mm, mean mm |", "|---|---|---|---|---|"]
for c in rows:
f = first.get(c["lead"])
was = "-" if f is None else f"{100 * f['p50']:.0f}%, {f['forecast_mean_mm']:.0f}"
lines.append(f"| {c['lead']} | {was} | {100 * c['blend_p50']:.0f}%, {100 * c['blend_p100']:.0f}%, {c['blend_mean_mm']:.0f} | "
f"{100 * c['extreme_share']:.0f}% | {100 * c['raw_p50']:.0f}%, {c['raw_mean_mm']:.0f} |")
return lines
def kept(place: str):
"""Whether the place's staged storms kept the hand-over (``None`` if its result does not say)."""
return R.kept(new[place])
def weak_rows() -> list:
"""Where the new Blend is still behind something, from the data."""
lines = []
behind = []
for place in new:
for g in G:
row = new[place]["groups"][g]
mine = F._value(row.get("stack"), "bss", "50")
rival, who = R.best_rival(row, "bss", "50", True)
if mine == mine and rival == rival and mine < rival:
behind.append((rival - mine, place, g, who))
behind.sort(reverse=True)
total = sum(1 for p in new for g in G if value(p, "stack", "bss", "50", g) == value(p, "stack", "bss", "50", g))
if behind:
worst = "; ".join(f"{SHORT[p]} {g} ({gap:.3f} behind {F.LABELS.get(str(w).replace('_raw', ''), w)}"
f"{' raw' if str(w).endswith('_raw') else ''})" for gap, p, g, w in behind[:4])
short = sum(1 for _, _, g, _ in behind if g in G[:2])
lines.append(f"- At 50 mm the Blend is behind some other system in {len(behind)} of {total} place and "
f"lead-group cells, {short} of them on days 1-7. The largest gaps: {worst}. Beyond a week the skill "
"of every system at 50 mm is close to zero and one storm can decide a cell.")
acc = []
for place in new:
row = new[place]["groups"][G[-1]]
mine = F._value(row.get("stack"), "acc", None)
rival, who = R.best_rival(row, "acc", None, True)
if mine == mine and rival == rival and mine < rival:
acc.append((rival - mine, place, mine, rival, who))
acc.sort(reverse=True)
if acc:
text = "; ".join(f"{SHORT[p]} {m:+.2f} against {r:+.2f} ({F.LABELS.get(str(w).replace('_raw', ''), w)}"
f"{' raw' if str(w).endswith('_raw') else ''})" for _, p, m, r, w in acc[:4])
lines.append(f"- On days 11-14 the anomaly correlation of the Blend's expected amount is behind another system "
f"in {len(acc)} of {len(new)} places: {text}. A "
"calibrated expected amount is smooth by construction, and a raw model's field keeps its pattern "
"even when the pattern is in the wrong place. A correction that traded RMSE for this score was "
"tested and not adopted ([WHAT_DID_NOT_WORK.md](WHAT_DID_NOT_WORK.md)).")
far = [(SHORT[p], value(p, "stack", "bss", "1", G[-1])) for p in new]
far = [(n, v) for n, v in far if v == v]
if far:
lines.append("- Days 11-14 carry little skill for anyone. Brier skill at 1 mm on those days: "
+ ", ".join(f"{n} {v:+.3f}" for n, v in far) + ". Zero is the climatology.")
off = [p for p in new if kept(p) is False]
if off:
lines.append(f"- In {names(off)} the staged storms were not forecast better with the hand-over, so it is off "
"there, and a storm beyond the record is forecast by the calibration alone, which shrinks it. The "
"table still flags such a forecast.")
agree, n = R.audit_agreement(new, G[0])
if n:
lines.append(f"- The verdict on the hand-over rests on four storms per place. It had the sign of what the "
f"hand-over then did in the test year in {agree} of {n} places. That is partly hindsight: the rule "
f"was fixed after {len(seen)} of the {len(new)} places had been scored. It can be overruled "
"(`pluvium blend train --handover always` or `never`).")
lines.append("- The online weights were scored with each day's rain known two days later. The satellite record is "
"published about four and a half months late, so a live system learns that much later"
+ (" ([what that costs](#with-the-rain-known-140-days-late))." if late else "."))
lines.append("- The quarterly refits used every run before each quarter. A live system can train only on runs "
"whose rain has been published, which are about five months old. That gap was not tested.")
lines.append("- The truth is one satellite product. CMORPH underestimates frozen and orographic precipitation "
"and the heaviest rain. Winter was scored against radar and gauges for the two US places only.")
lines.append("- The test year is the one on which the weak spots were found. A fresh year is the test that "
"counts, and it does not exist yet.")
return lines
def place_table(place: str) -> list:
res = new[place]
lines = [f"### {F.REGION_TITLES.get(place, place)}", "", f"{res['cells']:,} places.", "",
"| Score | " + " | ".join(G) + " |", "|---|" + "---|" * len(G)]
for title, key, t, higher in F.BOARD:
cells = []
for g in G:
row = res["groups"][g]
mine = F._value(row.get("stack"), key, t)
rival, who = R.best_rival(row, key, t, higher)
if mine != mine:
cells.append("-")
continue
ahead = rival != rival or (mine > rival if higher else mine < rival)
sign = "+" if key in ("bss", "acc") else ""
name = str(who)
label = F.LABELS.get(name.replace("_raw", ""), name) + (" raw" if name.endswith("_raw") else "")
cells.append(f"{'**' if ahead else ''}{mine:{sign}.3f}{'**' if ahead else ''} ({rival:{sign}.3f}, {label})")
lines.append(f"| {title} | " + " | ".join(cells) + " |")
weights = R.online_weights(res)
if weights:
lines += ["", "Online weights for light rain on days 1-3, at the end of each quarter of the test year:", ""] + weights
return lines + [""]
n_places = len(new)
c50 = count("bss", "50")
c50_short = count("bss", "50", groups=["days 1-3"])
c50_after = count("bss", "50", places=after)
c1 = count("bss", "1")
n_conf, better_conf, worse_conf = confident("50", G)
n1, better1, worse1 = confident("1", G[:2])
m50 = {g: (mean_of(old, "top", "bss", "50", g), mean_of(new, "stack", "bss", "50", g)) for g in G}
rmse = count("rmse", None, False)
acc = count("acc", None, True)
crps = count("crps", None, False)
on = [p for p in new if kept(p) is True]
off = [p for p in new if kept(p) is False]
agree_all, n_all = R.audit_agreement(new, G[0])
agree_after, n_after = R.audit_agreement(new, G[0], after)
agree_yearly, n_yearly = R.audit_agreement(new, G[0], rule="yearly")
lines = [
"# Pluvium Blend on the frontier: the second full-year benchmark", "",
"The first full-year benchmark ([FULL_YEAR_BENCHMARK.md](FULL_YEAR_BENCHMARK.md)) found three weak spots in Pluvium "
"Blend: the heavy tail (50 mm), winter in mid-latitude continental climates, and structure at long range. This "
"page scores a redesign aimed at them, on the same data, the same test year and the same rules.", "",
"## Read this first", "",
"The test year is the same as before (April 2025 to March 2026), and the weak spots were found on it. This is "
"a second look at a year already seen. Every design choice was made inside the training year (April 2024 to "
"March 2025), by cross-validation or by forecasting its later months from its earlier ones, and the design was "
"written down before the test year was scored. The parts are shown separately below, including where they cost "
"something.", "",
"One thing was changed after scoring began. The design as written down handed every record storm to the raw "
"systems. The first places scored showed that this helps in some places and does harm in others. Each place now "
"decides from its own training year: the hand-over is used only where it improved the forecast of storms "
"withheld from training. A first remedy, an audit on out-of-fold forecasts, was tried and dropped. Its verdict "
f"went against the test year in {n_yearly - agree_yearly} of {n_yearly} places, and its sign changes with "
f"which calibrated forecast it is computed on. The rule now used was fixed when {len(seen)} of the {len(new)} "
f"places had been scored ({names(seen)}). "
+ (f"Only {names(after)} {'was' if len(after) == 1 else 'were'} scored after it. " if after else "")
+ "The design as written down, the final one and the design with no hand-over are scored everywhere and shown "
"side by side.", "",
"One assumption of the test was too kind. The online weights were given each day's rain two days after it fell. "
"The satellite record is published about four and a half months late"
+ (", and [a later section](#with-the-rain-known-140-days-late) repeats the test with that delay." if late else "."),
"",
"The truth throughout is CMORPH, a satellite estimate, which is weak on frozen precipitation and on the heaviest "
"rain.", "",
"## What changed", "",
"1. Each post-processing network is trained on all rows with observed or forecast rain of 25 mm or more. The "
"other rows are thinned, with weights that undo the thinning.",
"2. The pool is fitted on the same kind of rows.",
"3. The components and the pool are re-weighed online after every verified day, with a 48-hour delay for the "
"truth to arrive and one learning rate, chosen on the training year.",
"4. A forecast outside anything in the training period is handed back, in part, to what the raw systems say.",
"5. Each place decides about that hand-over. The four wettest days of the training year are withheld from "
"training and then forecast, and the hand-over is used in a place only if it lowered the Brier score at 50 mm "
"on those storms.",
"6. Scenarios arrange the calibrated amounts in space like the ECMWF ensemble's members, so each scenario is as "
"varied as real weather.", "",
"## The short version", "",
]
summary = [
f"- At 50 mm, Brier skill improved on the first Blend in {c50['improved']} of {c50['cells']} place and "
f"lead-group cells, and in {c50_short['improved']} of {c50_short['cells']} on days 1-3. Mean skill over the places "
f"on days 1-3 went from {m50[G[0]][0]:+.4f} to {m50[G[0]][1]:+.4f}. The new Blend leads every other system at "
f"50 mm in {c50['new_leads']} of {c50['cells']} cells (the first led in {c50['old_leads']}).",
]
if after:
summary.append(f"- In the places scored after the design was final ({names(after)}), 50 mm skill improved on the "
f"first Blend in {c50_after['improved']} of {c50_after['cells']} cells, and the new Blend leads every "
f"other system in {c50_after['new_leads']} (the first: {c50_after['old_leads']}).")
summary += [
f"- Against each post-processed system at 50 mm, {better_conf} of {n_conf} comparisons are better with 95% "
f"confidence and {worse_conf} are worse.",
f"- At 1 mm, which is rain or no rain, the new Blend leads in {c1['new_leads']} of {c1['cells']} cells (the first: "
f"{c1['old_leads']}), and improved on the first in {c1['improved']}. On days 1-7, {better1} of {n1} comparisons "
f"with single post-processed systems are better with 95% confidence and {worse1} are worse.",
f"- The CRPS leads in {crps['new_leads']} of {crps['cells']} cells (was {crps['old_leads']}), the RMSE in "
f"{rmse['new_leads']} of {rmse['cells']} (was {rmse['old_leads']}) and the anomaly correlation in "
f"{acc['new_leads']} of {acc['cells']} (was {acc['old_leads']}).",
]
late1, late50 = count("bss", "1", groups=[G[-1]]), count("bss", "50", groups=[G[-1]])
if late1["new_leads"] < late1["old_leads"] or late50["new_leads"] < late50["old_leads"]:
summary.append(
f"- Days 11-14 got slightly worse. There the new Blend leads every other system at 1 mm in "
f"{late1['new_leads']} of {late1['cells']} places (the first: {late1['old_leads']}) and at 50 mm in "
f"{late50['new_leads']} (the first: {late50['old_leads']}). Mean Brier skill at 1 mm on those days went from "
f"{mean_of(old, 'top', 'bss', '1', G[-1]):+.4f} to {mean_of(new, 'stack', 'bss', '1', G[-1]):+.4f}. The pool "
f"alone scores {mean_of(new, 'pool', 'bss', '1', G[-1]):+.4f}: the online weights for that lead group learn "
"from few forecasts that verify late, and cost a little.")
if late:
g0 = G[0]
summary.append(
"- The online weights were scored with each day's rain known two days later, and the satellite record is "
f"published about 140 days late. With that delay, mean Brier skill on days 1-3 over {len(late)} places is "
f"{R.late_mean(new, late, '1', g0, 'late'):+.4f} at 1 mm and {R.late_mean(new, late, '50', g0, 'late'):+.4f} at "
f"50 mm. The figures above correspond to {R.late_mean(new, late, '1', g0, 'bench'):+.4f} and "
f"{R.late_mean(new, late, '50', g0, 'bench'):+.4f}, and with no online weights the Blend scores "
f"{R.late_mean(new, late, '1', g0, 'none'):+.4f} and {R.late_mean(new, late, '50', g0, 'none'):+.4f}.")
if n_all:
misses = R.audit_misses(new, G[0])
summary.append("- Each place's staged storms "
+ (f"kept the record-storm hand-over in {names(on)}" if on else "kept the record-storm hand-over nowhere")
+ (f" and switched it off in {names(off)}" if off else "")
+ f". That verdict had the sign of what the hand-over then did in the test year in {agree_all} of "
f"{n_all} places"
+ (f" ({agree_after} of the {n_after} scored after the rule was final)" if n_after else "")
+ f". The out-of-fold audit tried first would have been right in {agree_yearly} of {n_yearly}."
+ (" " + misses if misses else ""))
if scenarios:
def sc(kind, key, g):
vals = [r["groups"][g][kind][key] for r in scenarios.values() if r["groups"].get(g, {}).get(kind)]
return float(np.mean(vals)) if vals else float("nan")
last = G[-1]
summary.append(
f"- On days 11-14 one coupled scenario has {sc('coupled', 'sharpness', last):.2f} of "
f"the weather's spatial variability (raw ECMWF ENS: {sc('raw', 'sharpness', last):.2f}), and the CRPS of rain "
f"averaged over two-degree boxes is {sc('coupled', 'areal_crps', last):.2f} mm against "
f"{sc('uncoupled', 'areal_crps', last):.2f} for uncoupled draws and {sc('raw', 'areal_crps', last):.2f} for the "
f"raw ensemble (means over {len(scenarios)} places).")
lines += summary
lines.append("- The weak spots that remain are listed [near the end](#what-is-still-weak).")
lines += ["", "## Scoreboard", "",
f"In how many of the {n_places} places the new Blend beat every other system, raw and post-processed, on each "
"score. In brackets: the first Blend's count. Pluvium's own variants are not counted as rivals.", ""]
lines += board_rows()
lines += ["", "## Weak spot 1: the heavy tail", "",
"Brier skill score for 50 mm or more: first Blend -> new Blend (best other system). Bold where the new Blend "
"is ahead of every other system. A trailing + where it improved on the first.", ""]
lines += R.before_after(old, new, "bss", "50")
if tails:
lines += [""] + tail_bench.markdown([tails[k] for k in ORDER if k in tails]).splitlines()
lines += ["", "### The hand-over: staged, checked, and scored three ways", ""]
if withheld:
lines += ["Storms withheld from training (training year only). The wettest days were taken out of training, "
"then forecast. The calibrated forecast shrinks them. The hand-over gives some of that back where the raw "
"systems are the better guide, and does harm where they are not. Ordinary days are left alone. A place "
"uses the hand-over only where the Brier score in the fifth column fell (bold).", "",
"| Place | Withheld days | Observed mean, mm | Calibrated amount, mm | Brier at 50 mm: calibrated -> with hand-over | "
"Raw systems alone | Ordinary days handed over |", "|---|---|---|---|---|---|---|"]
for place, w in withheld.items():
label = "q0.999 w0.69"
s, c = w["storm"], w["control"]
better = s[label]["brier"]["50"] < s["calibrated"]["brier"]["50"]
lines.append(f"| {SHORT.get(place, place)} | {', '.join(w['storms'])} | {s['observed_mean_mm']:.1f} | "
f"{s['calibrated']['amount_mm']:.1f} | {s['calibrated']['brier']['50']:.4f} -> "
f"{'**' if better else ''}{s[label]['brier']['50']:.4f}{'**' if better else ''} | "
f"{s['raw']['brier']['50']:.4f} | {100 * c[label]['rows_handed_over']:.2f}% |")
lines.append("")
rules = R.rules_table(new, G[0], SHORT)
if rules:
lines += ["Two rules from training data, and what happened. The staged storms are the rule that decides. The "
"yearly check was tried first as the switch: the hand-over is applied to all the training period's "
"out-of-fold forecasts and the Brier score at 50 mm compared. It touches one row in a thousand, so its "
"effect on a year's score is a few tenths of a percent, and the sign of so small a number tells us "
"nothing. The last column is the test year: Brier skill at 50 mm on days 1-3 with the hand-over "
"always on, minus without it. A verdict is bold where it had the sign of the test year.", ""] + rules + [""]
lines += [f"The staged storms had the sign of the test year in {agree_all} of {n_all} places, the yearly check in "
f"{agree_yearly} of {n_yearly}. The staged rule was the first one written down, was replaced by the yearly "
f"check, and was restored when {len(seen)} of the {len(new)} places had been scored, so its count is partly "
"hindsight.", ""]
for g in G[:2]:
lines += [f"Brier skill at 50 mm in the test year, {g}, three ways. Bold is the best of the three.", ""]
lines += R.handover_three_ways(new, g, SHORT) + [""]
lines += ["## Weak spot 2: winter in mid-latitude climates", "",
"Brier skill score for 1 mm or more in each place's winter: first Blend -> new Blend (best other system and "
"which). Bold where the new Blend is ahead.", ""] + winter_rows()
if winters:
lines += ["", "### Winter against a second truth", "",
"CMORPH is weak on frozen precipitation, so a winter loss could be the forecast's or the truth's. The two "
"US places were scored again for December to February against MRMS, the US radar and gauge analysis, on "
"the same rows. Everything was trained on CMORPH, so every system carries the same handicap against "
"MRMS, and the question is whether the order between the systems changes.", ""]
labels = {k: v + ", post-processed" for k, v in F.LABELS.items()}
for place in ORDER:
if place in winters:
lines += winter_bench.markdown(winters[place], labels).splitlines() + [""]
if scenarios:
lines += ["", "## Weak spot 3: structure at long range", "",
"The map of expected amounts is smooth at long range, and should be: nobody knows where the rain band will "
"lie in twelve days. Its sharpness in the scoreboard's sense has not changed and will not. At that range "
"a user is better served by a set of scenarios that each look like weather, and those are scored here.", ""]
lines += scenario_bench.markdown([scenarios[k] for k in ORDER if k in scenarios]).splitlines()
lines += ["", "## What each part added", ""] + R.ablation(new)
mel = melissa_rows()
if mel:
lines += ["## Hurricane Melissa over Jamaica", ""] + mel + [""]
lines += R.rarity_lines(experiments_dir / "jamaica_returns.json")
if late:
cells, lower, below = R.late_counts(new, late)
where = f"all {n_places} places" if len(late) == n_places else f"{len(late)} of the {n_places} places ({names(late)})"
lines += ["## With the rain known 140 days late", "",
"The online weights learn from each day's rain, and in the test they were given it two days after it "
"fell. The record used as truth, NOAA's CMORPH climate data record, is not available that soon. On 15 "
"November 2026 its newest complete month was May 2026: the files for May had been written on 9 October "
"and those for June were still arriving. A live system therefore learns from each day about 140 days "
f"late. The test year was scored again with that delay and nothing else changed, in {where}. The delay "
"was applied to the warm-up over the training period as well, which is a little harsher than live use, "
"where all of the training period's rain is known by the time a blend is trained. The table gives the "
"mean Brier skill over those places.", ""] + R.late_rows(new, late) + [
"", f"With the delay, skill is lower than in the benchmark in {lower} of the {cells} cells (place, lead group, "
f"1 or 50 mm) and lower than with no online weights at all in {below}. The scoreboard hardly moves: the Blend "
f"is ahead of every other system at 1 mm in {R.late_leads(new, late, '1')[2]} of "
f"{R.late_leads(new, late, '1')[0]} cells (with the two-day delay: {R.late_leads(new, late, '1')[1]}) and at "
f"50 mm in {R.late_leads(new, late, '50')[2]} (with the two-day delay: {R.late_leads(new, late, '50')[1]}). "
"What the delay takes away is most of the small gain the online weights gave at 50 mm on days 1-3. The other "
"tables on this page keep the two-day delay, because that is the test as it was declared.", ""]
lines += ["## What is still weak", ""] + weak_rows() + [""]
lines += ["## Every place", "",
"Each cell: the new Blend (best other system and which). Bold where the Blend is ahead.", ""]
for place in ORDER:
if place in new:
lines += place_table(place)
out.write_text("\n".join(lines) + "\n")
print("wrote", out, len(lines), "lines;", n_places, "places")
# the same headline results, written into any other page that carries the markers (the manual, the long model card)
START, END = "<!-- frontier-summary:start -->", "<!-- frontier-summary:end -->"
block = [f"On the test year, in {n_places} places (the same year the first Blend was scored on):", ""] + summary
for page in (Path(p) for p in sys.argv[5:]):
text = page.read_text()
if START not in text or END not in text:
raise SystemExit(f"{page} has no {START} ... {END} markers")
head, rest = text.split(START, 1)
page.write_text(head + START + "\n" + "\n".join(block) + "\n" + END + rest.split(END, 1)[1])
print("updated the summary in", page)
|