odunbar commited on
Commit ·
27293d1
1
Parent(s): 245eff6
improve plotting
Browse files- src/common/leaderboard.py +110 -35
- src/data_store.py +29 -14
src/common/leaderboard.py
CHANGED
|
@@ -473,22 +473,60 @@ def render_leaderboard(
|
|
| 473 |
chart_df = chart_source.dropna(subset=["metric"]).groupby(
|
| 474 |
["abbreviation", "ensemble_size"], as_index=False
|
| 475 |
).agg(mean_forward_runs=("metric", "mean"))
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
)
|
| 490 |
-
|
| 491 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 492 |
|
| 493 |
# Mean-iterations-for-coverage section (UQ only, when budget_store provided)
|
| 494 |
if budget_store is not None and not budget_store.empty and selected_benchmark != "All":
|
|
@@ -499,28 +537,65 @@ def render_leaderboard(
|
|
| 499 |
bf = bf[bf["abbreviation"].isin(selected_methods)]
|
| 500 |
|
| 501 |
iters_df = bf[["abbreviation", "ensemble_size", "mean_iters"]].dropna(subset=["mean_iters"])
|
| 502 |
-
|
| 503 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 504 |
st.subheader("Mean Iterations for Coverage vs Ensemble Size")
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
.
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
|
|
|
|
|
|
|
|
|
| 521 |
)
|
| 522 |
-
|
| 523 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 524 |
|
| 525 |
# Failure panel — rendered regardless of whether the scored table has rows
|
| 526 |
if show_failure_panel and selected_benchmark != "All":
|
|
|
|
| 473 |
chart_df = chart_source.dropna(subset=["metric"]).groupby(
|
| 474 |
["abbreviation", "ensemble_size"], as_index=False
|
| 475 |
).agg(mean_forward_runs=("metric", "mean"))
|
| 476 |
+
|
| 477 |
+
all_ens_combos = chart_source[["abbreviation", "ensemble_size"]].drop_duplicates()
|
| 478 |
+
ens_ticks = sorted(all_ens_combos["ensemble_size"].unique().tolist()) if not all_ens_combos.empty else []
|
| 479 |
+
|
| 480 |
+
if not chart_df.empty:
|
| 481 |
+
_ok = chart_df[["abbreviation", "ensemble_size"]].assign(_ok=True)
|
| 482 |
+
fail_df = all_ens_combos.merge(_ok, on=["abbreviation", "ensemble_size"], how="left")
|
| 483 |
+
fail_df = fail_df[fail_df["_ok"].isna()].drop(columns="_ok").assign(mean_forward_runs=0.0)
|
| 484 |
+
else:
|
| 485 |
+
fail_df = all_ens_combos.assign(mean_forward_runs=0.0)
|
| 486 |
+
|
| 487 |
+
all_failed = chart_df.empty
|
| 488 |
+
chart_layers = []
|
| 489 |
+
if not chart_df.empty:
|
| 490 |
+
chart_layers.append(
|
| 491 |
+
alt.Chart(chart_df)
|
| 492 |
+
.mark_line(point=True)
|
| 493 |
+
.encode(
|
| 494 |
+
x=alt.X(
|
| 495 |
+
"ensemble_size:Q",
|
| 496 |
+
title="Ensemble Size",
|
| 497 |
+
axis=alt.Axis(values=ens_ticks, format="d"),
|
| 498 |
+
),
|
| 499 |
+
y=alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs"),
|
| 500 |
+
color=method_color,
|
| 501 |
+
tooltip=["abbreviation", "ensemble_size", alt.Tooltip("mean_forward_runs:Q", format=".4f")],
|
| 502 |
+
)
|
| 503 |
)
|
| 504 |
+
if not fail_df.empty:
|
| 505 |
+
y_fwd = (
|
| 506 |
+
alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs", scale=alt.Scale(domain=[0, 1]))
|
| 507 |
+
if all_failed
|
| 508 |
+
else alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs")
|
| 509 |
+
)
|
| 510 |
+
chart_layers.append(
|
| 511 |
+
alt.Chart(fail_df)
|
| 512 |
+
.mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0)
|
| 513 |
+
.encode(
|
| 514 |
+
x=alt.X(
|
| 515 |
+
"ensemble_size:Q",
|
| 516 |
+
title="Ensemble Size",
|
| 517 |
+
axis=alt.Axis(values=ens_ticks, format="d"),
|
| 518 |
+
),
|
| 519 |
+
y=y_fwd,
|
| 520 |
+
color=method_color,
|
| 521 |
+
tooltip=[
|
| 522 |
+
alt.Tooltip("abbreviation:N", title="Method"),
|
| 523 |
+
alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
|
| 524 |
+
alt.Tooltip("mean_forward_runs:Q", title="Value (all failed)"),
|
| 525 |
+
],
|
| 526 |
+
)
|
| 527 |
+
)
|
| 528 |
+
if chart_layers:
|
| 529 |
+
st.altair_chart(alt.layer(*chart_layers), use_container_width=True)
|
| 530 |
|
| 531 |
# Mean-iterations-for-coverage section (UQ only, when budget_store provided)
|
| 532 |
if budget_store is not None and not budget_store.empty and selected_benchmark != "All":
|
|
|
|
| 537 |
bf = bf[bf["abbreviation"].isin(selected_methods)]
|
| 538 |
|
| 539 |
iters_df = bf[["abbreviation", "ensemble_size", "mean_iters"]].dropna(subset=["mean_iters"])
|
| 540 |
+
all_ens_combos_iters = bf[["abbreviation", "ensemble_size"]].drop_duplicates()
|
| 541 |
+
if not all_ens_combos_iters.empty:
|
| 542 |
+
iters_ticks = sorted(all_ens_combos_iters["ensemble_size"].unique().tolist())
|
| 543 |
+
|
| 544 |
+
if not iters_df.empty:
|
| 545 |
+
_ok_iters = iters_df[["abbreviation", "ensemble_size"]].assign(_ok=True)
|
| 546 |
+
fail_df_iters = all_ens_combos_iters.merge(_ok_iters, on=["abbreviation", "ensemble_size"], how="left")
|
| 547 |
+
fail_df_iters = fail_df_iters[fail_df_iters["_ok"].isna()].drop(columns="_ok").assign(mean_iters=0.0)
|
| 548 |
+
else:
|
| 549 |
+
fail_df_iters = all_ens_combos_iters.assign(mean_iters=0.0)
|
| 550 |
+
|
| 551 |
+
all_failed_iters = iters_df.empty
|
| 552 |
st.subheader("Mean Iterations for Coverage vs Ensemble Size")
|
| 553 |
+
iters_layers = []
|
| 554 |
+
if not iters_df.empty:
|
| 555 |
+
iters_layers.append(
|
| 556 |
+
alt.Chart(iters_df)
|
| 557 |
+
.mark_line(point=True)
|
| 558 |
+
.encode(
|
| 559 |
+
x=alt.X(
|
| 560 |
+
"ensemble_size:Q",
|
| 561 |
+
title="Ensemble Size",
|
| 562 |
+
axis=alt.Axis(values=iters_ticks, format="d"),
|
| 563 |
+
),
|
| 564 |
+
y=alt.Y("mean_iters:Q", title="Mean Iterations"),
|
| 565 |
+
color=method_color,
|
| 566 |
+
tooltip=[
|
| 567 |
+
alt.Tooltip("abbreviation:N", title="Method"),
|
| 568 |
+
alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
|
| 569 |
+
alt.Tooltip("mean_iters:Q", format=".2f", title="Mean Iterations"),
|
| 570 |
+
],
|
| 571 |
+
)
|
| 572 |
)
|
| 573 |
+
if not fail_df_iters.empty:
|
| 574 |
+
y_iters = (
|
| 575 |
+
alt.Y("mean_iters:Q", title="Mean Iterations", scale=alt.Scale(domain=[0, 1]))
|
| 576 |
+
if all_failed_iters
|
| 577 |
+
else alt.Y("mean_iters:Q", title="Mean Iterations")
|
| 578 |
+
)
|
| 579 |
+
iters_layers.append(
|
| 580 |
+
alt.Chart(fail_df_iters)
|
| 581 |
+
.mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0)
|
| 582 |
+
.encode(
|
| 583 |
+
x=alt.X(
|
| 584 |
+
"ensemble_size:Q",
|
| 585 |
+
title="Ensemble Size",
|
| 586 |
+
axis=alt.Axis(values=iters_ticks, format="d"),
|
| 587 |
+
),
|
| 588 |
+
y=y_iters,
|
| 589 |
+
color=method_color,
|
| 590 |
+
tooltip=[
|
| 591 |
+
alt.Tooltip("abbreviation:N", title="Method"),
|
| 592 |
+
alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
|
| 593 |
+
alt.Tooltip("mean_iters:Q", title="Value (all failed)"),
|
| 594 |
+
],
|
| 595 |
+
)
|
| 596 |
+
)
|
| 597 |
+
if iters_layers:
|
| 598 |
+
st.altair_chart(alt.layer(*iters_layers), use_container_width=True)
|
| 599 |
|
| 600 |
# Failure panel — rendered regardless of whether the scored table has rows
|
| 601 |
if show_failure_panel and selected_benchmark != "All":
|
src/data_store.py
CHANGED
|
@@ -50,6 +50,10 @@ DATASET_FILES = {
|
|
| 50 |
],
|
| 51 |
}
|
| 52 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
# ---------------------------------------------------------------------------
|
| 54 |
# Uncertainty Quantification datasets
|
| 55 |
# ---------------------------------------------------------------------------
|
|
@@ -73,25 +77,16 @@ UQ_TARGET_LEVELS: list[float] = [1.0, 1.25, 1.5, 1.75, 2.0, 2.25, 2.5]
|
|
| 73 |
# Each NC must carry output_coverage (coverage_quantile, k_iter, ensemble_size, random_seed),
|
| 74 |
# target_scaling, and output_dim.
|
| 75 |
UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
|
| 76 |
-
"L63":
|
| 77 |
-
|
| 78 |
-
],
|
| 79 |
-
"
|
| 80 |
-
("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_ensemble_results_2026-06-15.nc"),
|
| 81 |
-
],
|
| 82 |
-
"L96_NN_FORCING": [
|
| 83 |
-
("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_nn_forcing_ensemble_results_2026-06-15.nc"),
|
| 84 |
-
],
|
| 85 |
-
"L96_SPATIAL_FORCING": [
|
| 86 |
-
("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_spatial_forcing_ensemble_results_2026-06-15.nc"),
|
| 87 |
-
],
|
| 88 |
}
|
| 89 |
|
| 90 |
# Quantile levels used for the all-quantiles-satisfied budget-for-coverage condition.
|
| 91 |
# Budget = N_ens · k_iter where first k s.t. |S(q)−q| ≤ c·√(q(1−q)/N_y) for ALL q below.
|
| 92 |
UQ_COVERAGE_QUANTILES: list[float] = [0.15, 0.5, 0.85]
|
| 93 |
|
| 94 |
-
EXPECTED_DIMS = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
|
| 95 |
|
| 96 |
|
| 97 |
# ---------------------------------------------------------------------------
|
|
@@ -135,6 +130,12 @@ def _load_store(dataset_files: dict[str, list[str]], target_col: str, *, drop_na
|
|
| 135 |
continue
|
| 136 |
|
| 137 |
metric = dataset["metric"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
|
| 139 |
# Track failures (metric == -1) as a percentage
|
| 140 |
failures_count = (metric == -1).sum(dim="random_seed")
|
|
@@ -279,11 +280,25 @@ def load_uq_budget_store() -> pd.DataFrame:
|
|
| 279 |
warnings.warn(f"{file_path.name}: missing output_coverage")
|
| 280 |
continue
|
| 281 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 282 |
cov_q = ds["coverage_quantile"].values # (n_cov_q,)
|
| 283 |
k_vals = ds["k_iter"].values # (n_k,) 1-indexed
|
| 284 |
ens_vals = ds["ensemble_size"].values # (n_ens,)
|
| 285 |
ts_vals = ds["target_scaling"].values # (n_ts,)
|
| 286 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
# dims: (coverage_quantile, k_iter, ensemble_size, random_seed)
|
| 288 |
cov_np = ds["output_coverage"].values
|
| 289 |
|
|
|
|
| 50 |
],
|
| 51 |
}
|
| 52 |
|
| 53 |
+
EXPECTED_DIMS_OPT = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
|
| 54 |
+
EXPECTED_DIMS_UQ = ("random_seed", "ensemble_size", "k_iter", "coverage_quantile")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
# ---------------------------------------------------------------------------
|
| 58 |
# Uncertainty Quantification datasets
|
| 59 |
# ---------------------------------------------------------------------------
|
|
|
|
| 77 |
# Each NC must carry output_coverage (coverage_quantile, k_iter, ensemble_size, random_seed),
|
| 78 |
# target_scaling, and output_dim.
|
| 79 |
UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
|
| 80 |
+
"L63": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l63_ensemble_results_2026-06-15_minimal.nc")],
|
| 81 |
+
"L96": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_ensemble_results_2026-06-15_minimal.nc")],
|
| 82 |
+
"L96_NN_FORCING": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_nn_forcing_ensemble_results_2026-06-15_minimal.nc")],
|
| 83 |
+
"L96_SPATIAL_FORCING": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_spatial_forcing_ensemble_results_2026-06-15_minimal.nc")],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
}
|
| 85 |
|
| 86 |
# Quantile levels used for the all-quantiles-satisfied budget-for-coverage condition.
|
| 87 |
# Budget = N_ens · k_iter where first k s.t. |S(q)−q| ≤ c·√(q(1−q)/N_y) for ALL q below.
|
| 88 |
UQ_COVERAGE_QUANTILES: list[float] = [0.15, 0.5, 0.85]
|
| 89 |
|
|
|
|
| 90 |
|
| 91 |
|
| 92 |
# ---------------------------------------------------------------------------
|
|
|
|
| 130 |
continue
|
| 131 |
|
| 132 |
metric = dataset["metric"]
|
| 133 |
+
if not set(metric.dims).issubset(EXPECTED_DIMS_OPT):
|
| 134 |
+
warnings.warn(
|
| 135 |
+
f"{file_path.name}: metric dims {metric.dims} contain unexpected "
|
| 136 |
+
f"dimensions not in EXPECTED_DIMS_OPT {EXPECTED_DIMS_OPT} — skipping"
|
| 137 |
+
)
|
| 138 |
+
continue
|
| 139 |
|
| 140 |
# Track failures (metric == -1) as a percentage
|
| 141 |
failures_count = (metric == -1).sum(dim="random_seed")
|
|
|
|
| 280 |
warnings.warn(f"{file_path.name}: missing output_coverage")
|
| 281 |
continue
|
| 282 |
|
| 283 |
+
if set(ds["output_coverage"].dims) != set(EXPECTED_DIMS_UQ):
|
| 284 |
+
warnings.warn(
|
| 285 |
+
f"{file_path.name}: output_coverage dims {ds['output_coverage'].dims} "
|
| 286 |
+
f"do not match EXPECTED_DIMS_UQ {EXPECTED_DIMS_UQ} — skipping"
|
| 287 |
+
)
|
| 288 |
+
continue
|
| 289 |
+
|
| 290 |
cov_q = ds["coverage_quantile"].values # (n_cov_q,)
|
| 291 |
k_vals = ds["k_iter"].values # (n_k,) 1-indexed
|
| 292 |
ens_vals = ds["ensemble_size"].values # (n_ens,)
|
| 293 |
ts_vals = ds["target_scaling"].values # (n_ts,)
|
| 294 |
+
# xarray drops dimensions that no variable uses; read output_dim
|
| 295 |
+
# directly from the underlying netCDF4 file to handle minimal files.
|
| 296 |
+
nc4 = importlib.import_module("netCDF4")
|
| 297 |
+
with nc4.Dataset(file_path) as _nc:
|
| 298 |
+
if "output_dim" not in _nc.dimensions:
|
| 299 |
+
warnings.warn(f"{file_path.name}: missing output_dim dimension")
|
| 300 |
+
continue
|
| 301 |
+
n_y = len(_nc.dimensions["output_dim"])
|
| 302 |
# dims: (coverage_quantile, k_iter, ensemble_size, random_seed)
|
| 303 |
cov_np = ds["output_coverage"].values
|
| 304 |
|