odunbar commited on
Commit
27293d1
·
1 Parent(s): 245eff6

improve plotting

Browse files
Files changed (2) hide show
  1. src/common/leaderboard.py +110 -35
  2. src/data_store.py +29 -14
src/common/leaderboard.py CHANGED
@@ -473,22 +473,60 @@ def render_leaderboard(
473
  chart_df = chart_source.dropna(subset=["metric"]).groupby(
474
  ["abbreviation", "ensemble_size"], as_index=False
475
  ).agg(mean_forward_runs=("metric", "mean"))
476
- ens_ticks = sorted(chart_df["ensemble_size"].unique().tolist())
477
- chart = (
478
- alt.Chart(chart_df)
479
- .mark_line(point=True)
480
- .encode(
481
- x=alt.X(
482
- "ensemble_size:Q",
483
- title="Ensemble Size",
484
- axis=alt.Axis(values=ens_ticks, format="d"),
485
- ),
486
- y=alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs"),
487
- color=method_color,
488
- tooltip=["abbreviation", "ensemble_size", alt.Tooltip("mean_forward_runs:Q", format=".4f")],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
489
  )
490
- )
491
- st.altair_chart(chart, use_container_width=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
492
 
493
  # Mean-iterations-for-coverage section (UQ only, when budget_store provided)
494
  if budget_store is not None and not budget_store.empty and selected_benchmark != "All":
@@ -499,28 +537,65 @@ def render_leaderboard(
499
  bf = bf[bf["abbreviation"].isin(selected_methods)]
500
 
501
  iters_df = bf[["abbreviation", "ensemble_size", "mean_iters"]].dropna(subset=["mean_iters"])
502
- if not iters_df.empty:
503
- iters_ticks = sorted(iters_df["ensemble_size"].unique().tolist())
 
 
 
 
 
 
 
 
 
 
504
  st.subheader("Mean Iterations for Coverage vs Ensemble Size")
505
- iters_chart = (
506
- alt.Chart(iters_df)
507
- .mark_line(point=True)
508
- .encode(
509
- x=alt.X(
510
- "ensemble_size:Q",
511
- title="Ensemble Size",
512
- axis=alt.Axis(values=iters_ticks, format="d"),
513
- ),
514
- y=alt.Y("mean_iters:Q", title="Mean Iterations"),
515
- color=method_color,
516
- tooltip=[
517
- alt.Tooltip("abbreviation:N", title="Method"),
518
- alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
519
- alt.Tooltip("mean_iters:Q", format=".2f", title="Mean Iterations"),
520
- ],
 
 
 
521
  )
522
- )
523
- st.altair_chart(iters_chart, use_container_width=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
524
 
525
  # Failure panel — rendered regardless of whether the scored table has rows
526
  if show_failure_panel and selected_benchmark != "All":
 
473
  chart_df = chart_source.dropna(subset=["metric"]).groupby(
474
  ["abbreviation", "ensemble_size"], as_index=False
475
  ).agg(mean_forward_runs=("metric", "mean"))
476
+
477
+ all_ens_combos = chart_source[["abbreviation", "ensemble_size"]].drop_duplicates()
478
+ ens_ticks = sorted(all_ens_combos["ensemble_size"].unique().tolist()) if not all_ens_combos.empty else []
479
+
480
+ if not chart_df.empty:
481
+ _ok = chart_df[["abbreviation", "ensemble_size"]].assign(_ok=True)
482
+ fail_df = all_ens_combos.merge(_ok, on=["abbreviation", "ensemble_size"], how="left")
483
+ fail_df = fail_df[fail_df["_ok"].isna()].drop(columns="_ok").assign(mean_forward_runs=0.0)
484
+ else:
485
+ fail_df = all_ens_combos.assign(mean_forward_runs=0.0)
486
+
487
+ all_failed = chart_df.empty
488
+ chart_layers = []
489
+ if not chart_df.empty:
490
+ chart_layers.append(
491
+ alt.Chart(chart_df)
492
+ .mark_line(point=True)
493
+ .encode(
494
+ x=alt.X(
495
+ "ensemble_size:Q",
496
+ title="Ensemble Size",
497
+ axis=alt.Axis(values=ens_ticks, format="d"),
498
+ ),
499
+ y=alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs"),
500
+ color=method_color,
501
+ tooltip=["abbreviation", "ensemble_size", alt.Tooltip("mean_forward_runs:Q", format=".4f")],
502
+ )
503
  )
504
+ if not fail_df.empty:
505
+ y_fwd = (
506
+ alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs", scale=alt.Scale(domain=[0, 1]))
507
+ if all_failed
508
+ else alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs")
509
+ )
510
+ chart_layers.append(
511
+ alt.Chart(fail_df)
512
+ .mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0)
513
+ .encode(
514
+ x=alt.X(
515
+ "ensemble_size:Q",
516
+ title="Ensemble Size",
517
+ axis=alt.Axis(values=ens_ticks, format="d"),
518
+ ),
519
+ y=y_fwd,
520
+ color=method_color,
521
+ tooltip=[
522
+ alt.Tooltip("abbreviation:N", title="Method"),
523
+ alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
524
+ alt.Tooltip("mean_forward_runs:Q", title="Value (all failed)"),
525
+ ],
526
+ )
527
+ )
528
+ if chart_layers:
529
+ st.altair_chart(alt.layer(*chart_layers), use_container_width=True)
530
 
531
  # Mean-iterations-for-coverage section (UQ only, when budget_store provided)
532
  if budget_store is not None and not budget_store.empty and selected_benchmark != "All":
 
537
  bf = bf[bf["abbreviation"].isin(selected_methods)]
538
 
539
  iters_df = bf[["abbreviation", "ensemble_size", "mean_iters"]].dropna(subset=["mean_iters"])
540
+ all_ens_combos_iters = bf[["abbreviation", "ensemble_size"]].drop_duplicates()
541
+ if not all_ens_combos_iters.empty:
542
+ iters_ticks = sorted(all_ens_combos_iters["ensemble_size"].unique().tolist())
543
+
544
+ if not iters_df.empty:
545
+ _ok_iters = iters_df[["abbreviation", "ensemble_size"]].assign(_ok=True)
546
+ fail_df_iters = all_ens_combos_iters.merge(_ok_iters, on=["abbreviation", "ensemble_size"], how="left")
547
+ fail_df_iters = fail_df_iters[fail_df_iters["_ok"].isna()].drop(columns="_ok").assign(mean_iters=0.0)
548
+ else:
549
+ fail_df_iters = all_ens_combos_iters.assign(mean_iters=0.0)
550
+
551
+ all_failed_iters = iters_df.empty
552
  st.subheader("Mean Iterations for Coverage vs Ensemble Size")
553
+ iters_layers = []
554
+ if not iters_df.empty:
555
+ iters_layers.append(
556
+ alt.Chart(iters_df)
557
+ .mark_line(point=True)
558
+ .encode(
559
+ x=alt.X(
560
+ "ensemble_size:Q",
561
+ title="Ensemble Size",
562
+ axis=alt.Axis(values=iters_ticks, format="d"),
563
+ ),
564
+ y=alt.Y("mean_iters:Q", title="Mean Iterations"),
565
+ color=method_color,
566
+ tooltip=[
567
+ alt.Tooltip("abbreviation:N", title="Method"),
568
+ alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
569
+ alt.Tooltip("mean_iters:Q", format=".2f", title="Mean Iterations"),
570
+ ],
571
+ )
572
  )
573
+ if not fail_df_iters.empty:
574
+ y_iters = (
575
+ alt.Y("mean_iters:Q", title="Mean Iterations", scale=alt.Scale(domain=[0, 1]))
576
+ if all_failed_iters
577
+ else alt.Y("mean_iters:Q", title="Mean Iterations")
578
+ )
579
+ iters_layers.append(
580
+ alt.Chart(fail_df_iters)
581
+ .mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0)
582
+ .encode(
583
+ x=alt.X(
584
+ "ensemble_size:Q",
585
+ title="Ensemble Size",
586
+ axis=alt.Axis(values=iters_ticks, format="d"),
587
+ ),
588
+ y=y_iters,
589
+ color=method_color,
590
+ tooltip=[
591
+ alt.Tooltip("abbreviation:N", title="Method"),
592
+ alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
593
+ alt.Tooltip("mean_iters:Q", title="Value (all failed)"),
594
+ ],
595
+ )
596
+ )
597
+ if iters_layers:
598
+ st.altair_chart(alt.layer(*iters_layers), use_container_width=True)
599
 
600
  # Failure panel — rendered regardless of whether the scored table has rows
601
  if show_failure_panel and selected_benchmark != "All":
src/data_store.py CHANGED
@@ -50,6 +50,10 @@ DATASET_FILES = {
50
  ],
51
  }
52
 
 
 
 
 
53
  # ---------------------------------------------------------------------------
54
  # Uncertainty Quantification datasets
55
  # ---------------------------------------------------------------------------
@@ -73,25 +77,16 @@ UQ_TARGET_LEVELS: list[float] = [1.0, 1.25, 1.5, 1.75, 2.0, 2.25, 2.5]
73
  # Each NC must carry output_coverage (coverage_quantile, k_iter, ensemble_size, random_seed),
74
  # target_scaling, and output_dim.
75
  UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
76
- "L63": [
77
- ("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l63_ensemble_results_2026-06-15.nc"),
78
- ],
79
- "L96": [
80
- ("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_ensemble_results_2026-06-15.nc"),
81
- ],
82
- "L96_NN_FORCING": [
83
- ("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_nn_forcing_ensemble_results_2026-06-15.nc"),
84
- ],
85
- "L96_SPATIAL_FORCING": [
86
- ("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_spatial_forcing_ensemble_results_2026-06-15.nc"),
87
- ],
88
  }
89
 
90
  # Quantile levels used for the all-quantiles-satisfied budget-for-coverage condition.
91
  # Budget = N_ens · k_iter where first k s.t. |S(q)−q| ≤ c·√(q(1−q)/N_y) for ALL q below.
92
  UQ_COVERAGE_QUANTILES: list[float] = [0.15, 0.5, 0.85]
93
 
94
- EXPECTED_DIMS = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
95
 
96
 
97
  # ---------------------------------------------------------------------------
@@ -135,6 +130,12 @@ def _load_store(dataset_files: dict[str, list[str]], target_col: str, *, drop_na
135
  continue
136
 
137
  metric = dataset["metric"]
 
 
 
 
 
 
138
 
139
  # Track failures (metric == -1) as a percentage
140
  failures_count = (metric == -1).sum(dim="random_seed")
@@ -279,11 +280,25 @@ def load_uq_budget_store() -> pd.DataFrame:
279
  warnings.warn(f"{file_path.name}: missing output_coverage")
280
  continue
281
 
 
 
 
 
 
 
 
282
  cov_q = ds["coverage_quantile"].values # (n_cov_q,)
283
  k_vals = ds["k_iter"].values # (n_k,) 1-indexed
284
  ens_vals = ds["ensemble_size"].values # (n_ens,)
285
  ts_vals = ds["target_scaling"].values # (n_ts,)
286
- n_y = ds.sizes["output_dim"]
 
 
 
 
 
 
 
287
  # dims: (coverage_quantile, k_iter, ensemble_size, random_seed)
288
  cov_np = ds["output_coverage"].values
289
 
 
50
  ],
51
  }
52
 
53
+ EXPECTED_DIMS_OPT = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
54
+ EXPECTED_DIMS_UQ = ("random_seed", "ensemble_size", "k_iter", "coverage_quantile")
55
+
56
+
57
  # ---------------------------------------------------------------------------
58
  # Uncertainty Quantification datasets
59
  # ---------------------------------------------------------------------------
 
77
  # Each NC must carry output_coverage (coverage_quantile, k_iter, ensemble_size, random_seed),
78
  # target_scaling, and output_dim.
79
  UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
80
+ "L63": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l63_ensemble_results_2026-06-15_minimal.nc")],
81
+ "L96": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_ensemble_results_2026-06-15_minimal.nc")],
82
+ "L96_NN_FORCING": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_nn_forcing_ensemble_results_2026-06-15_minimal.nc")],
83
+ "L96_SPATIAL_FORCING": [("ces-eki-dmc", "ces-eki-dmc_results/ces-eki-dmc_l96_spatial_forcing_ensemble_results_2026-06-15_minimal.nc")],
 
 
 
 
 
 
 
 
84
  }
85
 
86
  # Quantile levels used for the all-quantiles-satisfied budget-for-coverage condition.
87
  # Budget = N_ens · k_iter where first k s.t. |S(q)−q| ≤ c·√(q(1−q)/N_y) for ALL q below.
88
  UQ_COVERAGE_QUANTILES: list[float] = [0.15, 0.5, 0.85]
89
 
 
90
 
91
 
92
  # ---------------------------------------------------------------------------
 
130
  continue
131
 
132
  metric = dataset["metric"]
133
+ if not set(metric.dims).issubset(EXPECTED_DIMS_OPT):
134
+ warnings.warn(
135
+ f"{file_path.name}: metric dims {metric.dims} contain unexpected "
136
+ f"dimensions not in EXPECTED_DIMS_OPT {EXPECTED_DIMS_OPT} — skipping"
137
+ )
138
+ continue
139
 
140
  # Track failures (metric == -1) as a percentage
141
  failures_count = (metric == -1).sum(dim="random_seed")
 
280
  warnings.warn(f"{file_path.name}: missing output_coverage")
281
  continue
282
 
283
+ if set(ds["output_coverage"].dims) != set(EXPECTED_DIMS_UQ):
284
+ warnings.warn(
285
+ f"{file_path.name}: output_coverage dims {ds['output_coverage'].dims} "
286
+ f"do not match EXPECTED_DIMS_UQ {EXPECTED_DIMS_UQ} — skipping"
287
+ )
288
+ continue
289
+
290
  cov_q = ds["coverage_quantile"].values # (n_cov_q,)
291
  k_vals = ds["k_iter"].values # (n_k,) 1-indexed
292
  ens_vals = ds["ensemble_size"].values # (n_ens,)
293
  ts_vals = ds["target_scaling"].values # (n_ts,)
294
+ # xarray drops dimensions that no variable uses; read output_dim
295
+ # directly from the underlying netCDF4 file to handle minimal files.
296
+ nc4 = importlib.import_module("netCDF4")
297
+ with nc4.Dataset(file_path) as _nc:
298
+ if "output_dim" not in _nc.dimensions:
299
+ warnings.warn(f"{file_path.name}: missing output_dim dimension")
300
+ continue
301
+ n_y = len(_nc.dimensions["output_dim"])
302
  # dims: (coverage_quantile, k_iter, ensemble_size, random_seed)
303
  cov_np = ds["output_coverage"].values
304