Separate GBOED from IES and rerun

#19
by odunbar - opened
Files changed (27) hide show
  1. .cache/known_methods_snapshot.json +47 -6
  2. .claude/skills/leaderboard-manager/references/manage-display.md +10 -25
  3. CLAUDE.md +27 -2
  4. data/{gboed-uq_results/boed_l63_ensemble_results_2026-07-30_minimal.nc → adam_results/leaderboard_adam_l96_flux-force_2026-10-21.nc} +2 -2
  5. data/gboed-uq_results/{boed_l96_const-force_2026-07-30_minimal.nc → boed-iekf-eig_l63_ensemble_results_2026-10-19_minimal.nc} +2 -2
  6. data/gboed-uq_results/{boed_l96_vec-force_2026-08-07_minimal.nc → boed-iekf-eig_l96_const-force_2026-10-19_minimal.nc} +2 -2
  7. data/gboed-uq_results/boed-iekf-eig_l96_flux-force_2026-10-19_minimal.nc +3 -0
  8. data/gboed-uq_results/boed-iekf-eig_l96_vec-force_2026-10-19_minimal.nc +3 -0
  9. data/gboed-uq_results/boed-iekf_l63_ensemble_results_2026-10-19_minimal.nc +3 -0
  10. data/gboed-uq_results/boed-iekf_l96_const-force_2026-10-19_minimal.nc +3 -0
  11. data/gboed-uq_results/boed-iekf_l96_flux-force_2026-10-19_minimal.nc +3 -0
  12. data/gboed-uq_results/boed-iekf_l96_vec-force_2026-10-19_minimal.nc +3 -0
  13. data/gboed-uq_results/boed-tmcmc-eig_l63_ensemble_results_2026-10-16_minimal.nc +3 -0
  14. data/gboed-uq_results/boed-tmcmc-eig_l96_const-force_2026-10-16_minimal.nc +3 -0
  15. data/gboed-uq_results/boed-tmcmc-eig_l96_flux-force_2026-10-16_minimal.nc +3 -0
  16. data/gboed-uq_results/boed-tmcmc-eig_l96_vec-force_2026-10-16_minimal.nc +3 -0
  17. data/gboed-uq_results/boed-tmcmc_l63_ensemble_results_2026-10-16_minimal.nc +3 -0
  18. data/gboed-uq_results/boed-tmcmc_l96_const-force_2026-10-16_minimal.nc +3 -0
  19. data/gboed-uq_results/boed-tmcmc_l96_flux-force_2026-10-16_minimal.nc +3 -0
  20. data/gboed-uq_results/boed-tmcmc_l96_vec-force_2026-10-16_minimal.nc +3 -0
  21. data/levenberg_marquardt_results/leaderboard_lm_l96_flux-force_2026-10-20.nc +3 -0
  22. src/common/leaderboard.py +86 -120
  23. src/common/method_registry.py +66 -4
  24. src/data_store.py +18 -3
  25. src/pages/MethodDetails.py +135 -3
  26. src/pages/UQLeaderboard.py +0 -1
  27. src/streamlit_app.py +14 -2
.cache/known_methods_snapshot.json CHANGED
@@ -126,16 +126,20 @@
126
  "gradient_descent"
127
  ]
128
  },
129
- "gboed": {
130
- "abbreviation": "GBOED",
131
- "Method": "Goal-oriented Bayesian Optimal Experimental Design",
132
  "parallelism": "parallel-interacting",
133
  "update_type": "general",
134
  "method_goal": "uq",
135
  "emulator_use": "within-optimize",
136
  "aliases": [
137
  "gboed",
138
- "boed"
 
 
 
 
139
  ]
140
  },
141
  "lm-kgmm": {
@@ -193,7 +197,7 @@
193
  "rl-ddpg-short": {
194
  "abbreviation": "RL-DDPG-SHORT",
195
  "Method": "Reinforcement Learning (DDPG, short RL step)",
196
- "parallelism": "serial",
197
  "update_type": "gradient (score)",
198
  "method_goal": "optimization",
199
  "emulator_use": "none",
@@ -216,7 +220,7 @@
216
  "rl-td3-short": {
217
  "abbreviation": "RL-TD3-SHORT",
218
  "Method": "Reinforcement Learning (TD3, short RL step)",
219
- "parallelism": "serial",
220
  "update_type": "gradient (score)",
221
  "method_goal": "optimization",
222
  "emulator_use": "none",
@@ -235,6 +239,43 @@
235
  "rl-td3",
236
  "rl-td3-long"
237
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
238
  }
239
  },
240
  "observed_methods": [
 
126
  "gradient_descent"
127
  ]
128
  },
129
+ "gboed-tmcmc": {
130
+ "abbreviation": "GBOED-TMCMC",
131
+ "Method": "Goal-oriented Bayesian Optimal Experimental Design (TMCMC)",
132
  "parallelism": "parallel-interacting",
133
  "update_type": "general",
134
  "method_goal": "uq",
135
  "emulator_use": "within-optimize",
136
  "aliases": [
137
  "gboed",
138
+ "gboed-tmcmc",
139
+ "boed",
140
+ "gboed-tmcmc-eig",
141
+ "boed-tmcmc-eig",
142
+ "tmcmc-eig"
143
  ]
144
  },
145
  "lm-kgmm": {
 
197
  "rl-ddpg-short": {
198
  "abbreviation": "RL-DDPG-SHORT",
199
  "Method": "Reinforcement Learning (DDPG, short RL step)",
200
+ "parallelism": "serial (short window)",
201
  "update_type": "gradient (score)",
202
  "method_goal": "optimization",
203
  "emulator_use": "none",
 
220
  "rl-td3-short": {
221
  "abbreviation": "RL-TD3-SHORT",
222
  "Method": "Reinforcement Learning (TD3, short RL step)",
223
+ "parallelism": "serial (short window)",
224
  "update_type": "gradient (score)",
225
  "method_goal": "optimization",
226
  "emulator_use": "none",
 
239
  "rl-td3",
240
  "rl-td3-long"
241
  ]
242
+ },
243
+ "ies-iekf": {
244
+ "abbreviation": "IES-IEKF",
245
+ "Method": "Iterated Emulate-Sample (IEKF)",
246
+ "parallelism": "parallel-interacting",
247
+ "update_type": "general",
248
+ "method_goal": "uq",
249
+ "emulator_use": "within-optimize",
250
+ "aliases": [
251
+ "ies-iekf",
252
+ "boed-iekf"
253
+ ]
254
+ },
255
+ "gboed-iekf": {
256
+ "abbreviation": "GBOED-IEKF",
257
+ "Method": "Goal-oriented Bayesian Optimal Experimental Design (IEKF)",
258
+ "parallelism": "parallel-interacting",
259
+ "update_type": "general",
260
+ "method_goal": "uq",
261
+ "emulator_use": "within-optimize",
262
+ "aliases": [
263
+ "gboed-iekf",
264
+ "gboed-iekf-eig",
265
+ "boed-iekf-eig"
266
+ ]
267
+ },
268
+ "ies-tmcmc": {
269
+ "abbreviation": "IES-TMCMC",
270
+ "Method": "Iterated Emulate-Sample (TMCMC)",
271
+ "parallelism": "parallel-interacting",
272
+ "update_type": "general",
273
+ "method_goal": "uq",
274
+ "emulator_use": "within-optimize",
275
+ "aliases": [
276
+ "ies-tmcmc",
277
+ "boed-tmcmc"
278
+ ]
279
  }
280
  },
281
  "observed_methods": [
.claude/skills/leaderboard-manager/references/manage-display.md CHANGED
@@ -23,9 +23,8 @@ next time someone edits the file.
23
  2. Benchmark selector (`st.selectbox("Benchmark", ...)`) — always a single benchmark;
24
  there is **no "All benchmarks" option**. Every chart and the table below operate on
25
  whatever `selected_benchmark` is currently chosen.
26
- 3. **Controls — always visible**, not collapsed. Look for `st.subheader("Scoring &
27
- Target Controls")`: target-level radio, scoring-mode radio (+ blend-weight slider
28
- when "Custom Blend" is selected), and the "Methods to display in charts" pills
29
  selector (see "Methods selector" below). These used to be wrapped in
30
  `st.expander(..., expanded=False)`; that was removed so users don't have to open a
31
  dropdown to see or change the target level.
@@ -67,7 +66,7 @@ list (not one-per-view):
67
  table_column_order = [
68
  "Placement", "abbreviation", "Method",
69
  "update_type", "parallelism", "method_goal", "emulator_use", "Score",
70
- "Mean Forward Model Runs", "Minimum Forward Model Runs",
71
  "Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
72
  ]
73
  ```
@@ -106,28 +105,14 @@ To add it:
106
 
107
  ---
108
 
109
- ## Scoring modes
110
-
111
- The four scoring modes are defined in `scoring_options` (a list literal near the top of
112
- `render_leaderboard()`, right after `target_options` is computed):
113
- ```python
114
- scoring_options = [
115
- "Mean Forward Model Runs",
116
- "Minimum Forward Model Runs",
117
- "Smallest Optimal Ensemble Size",
118
- "Custom Blend",
119
- ]
120
- ```
121
-
122
- Each mode maps to a sort key and a score column assignment inside `build_scored_table()`
123
- — search for the `if scoring_mode == "Mean Forward Model Runs": ... elif ...` chain. To
124
- **add a new scoring mode**, add its name to `scoring_options` and add an `elif` branch in
125
- that chain that sets `scored_df["Score"]`, `sort_columns`, and `ascending`.
126
 
127
- The "Custom Blend" slider (`st.slider("Blend Weight: Forward Runs vs Ensemble Size", ...)`)
128
- is keyed to the session-state key `k_weight` and only rendered when
129
- `st.session_state[k_scoring] == "Custom Blend"` — no change needed when adding a
130
- non-blend mode. It lives in the always-visible controls section, not behind an expander.
 
 
131
 
132
  ---
133
 
 
23
  2. Benchmark selector (`st.selectbox("Benchmark", ...)`) — always a single benchmark;
24
  there is **no "All benchmarks" option**. Every chart and the table below operate on
25
  whatever `selected_benchmark` is currently chosen.
26
+ 3. **Controls — always visible**, not collapsed. Look for `st.subheader("Target &
27
+ Method Controls")`: target-level radio and the "Methods to display in charts" pills
 
28
  selector (see "Methods selector" below). These used to be wrapped in
29
  `st.expander(..., expanded=False)`; that was removed so users don't have to open a
30
  dropdown to see or change the target level.
 
66
  table_column_order = [
67
  "Placement", "abbreviation", "Method",
68
  "update_type", "parallelism", "method_goal", "emulator_use", "Score",
69
+ "Mean Forward Model Runs",
70
  "Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
71
  ]
72
  ```
 
105
 
106
  ---
107
 
108
+ ## Scoring
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
 
110
+ There is a single scoring basis: mean forward-model runs. The scoring-mode radio, the
111
+ "Custom Blend" slider and the minimum-runs / ensemble-size modes were removed. In
112
+ `build_scored_table()`, `Score` is the normalized 0-100 form of `Mean Forward Model Runs`
113
+ (`mean_runs_score`), and rows are sorted by reliability, then mean runs, then optimal
114
+ ensemble size. To reintroduce another mode, add a control above the target radio and branch
115
+ on it where `Score` / `sort_columns` are set.
116
 
117
  ---
118
 
CLAUDE.md CHANGED
@@ -57,6 +57,8 @@ calibration_benchmark/
57
  │ │ │ when budget_store is None (Optimization board), instead renders a
58
  │ │ │ "Mean Iterations to Reach Target" chart derived as metric/ensemble_size
59
  │ │ └── method_registry.py KNOWN_METHODS registry + name canonicalization
 
 
60
  │ └── pages/
61
  │ ├── UQLeaderboard.py 🎯 UQ Leaderboard — calls render_leaderboard(load_uq_store())
62
  │ ├── MethodDetails.py 📘 Method configurations — per-method (selection-driven) technical
@@ -81,9 +83,19 @@ calibration_benchmark/
81
  │ ├── rl-td3/ RL-TD3-SHORT / RL-TD3 (TD3 variant of the RL experiments; RL-TD3 files are `rl-td3-long` in the NetCDF) results (per-benchmark files)
82
  │ ├── rl-ddpg/ RL-DDPG-SHORT / RL-DDPG (DDPG actor-critic; short RL step / standard-window RL step; RL-DDPG files are `rl-ddpg-long` in the NetCDF) results (per-benchmark files)
83
  │ ├── ces-eki-dmc_results/ CES-EKI-DMC ensemble-results files (UQ source)
84
- │ ├── gboed-uq_results/ GBOED ensemble-results files (UQ source)
85
  │ ├── gnki-uq_results/ IEKF (a.k.a GNKI) ensemble-results files (UQ source)
86
  │ └── abc/ ABC ensemble-results files (UQ source; per-benchmark files)
 
 
 
 
 
 
 
 
 
 
87
  ├── examples/ lorenz_demo.ipynb + lorenz.py (reference, not app code)
88
  ├── From_Rob_6-10/ Raw collaborator data drop — NOT read by the app
89
  ├── .cache/ known_methods_snapshot.json (auto-generated, safe to delete)
@@ -204,7 +216,10 @@ A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
204
  | `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | gradient (Kalman) | uq | after-optimize | — |
205
  | `adam` | ADAM | serial | gradient (exact) | optimization | none | — |
206
  | `lm` | LM | serial | gradient (exact) | optimization | none | gradient_descent |
207
- | `gboed` | GBOED | parallel-interacting | general | uq | within-optimize | boed |
 
 
 
208
  | `lm-kgmm` | LM-KGMM | serial | gradient (score) | optimization | none | — |
209
  | `lm-dsm` | LM-DSM | serial | gradient (score) | optimization | none | — |
210
  | `adam-kgmm` | ADAM-KGMM | serial | gradient (score) | optimization | none | — |
@@ -223,6 +238,10 @@ A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
223
  2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
224
  keyed by **abbreviation** (e.g. `"TEKI"`). ⚠️ Metadata is split across both files.
225
  3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
 
 
 
 
226
 
227
  ### Add an optimization benchmark dataset
228
  1. Add a `"BENCHMARK_NAME": [list_of_nc_paths]` entry to `DATASET_FILES` in `src/data_store.py`.
@@ -295,6 +314,12 @@ the `budget_store` parameter of `render_leaderboard()` in `src/common/leaderboar
295
  - **Suitability table is fixed at `default_target`** — `_render_suitability_table()` always
296
  evaluates suitability at the `default_target` passed to `render_leaderboard()`, independent
297
  of what the user selects in the target radio control below it.
 
 
 
 
 
 
298
  - **Suitability criteria** — a method is green for a benchmark if there exists an ensemble size
299
  with `failure_rate < 20%` and `mean_budget ≤ 3× global_best` at `default_target`. Gray means
300
  no data for that benchmark; the method may be suitable on other benchmarks.
 
57
  │ │ │ when budget_store is None (Optimization board), instead renders a
58
  │ │ │ "Mean Iterations to Reach Target" chart derived as metric/ensemble_size
59
  │ │ └── method_registry.py KNOWN_METHODS registry + name canonicalization
60
+ │ │ SUITABILITY_FAMILIES — method families (CES, Kalman, Gradient, GFDT, RL, IES,
61
+ │ │ GBOED, General) grouping rows in the online + LaTeX suitability tables
62
  │ └── pages/
63
  │ ├── UQLeaderboard.py 🎯 UQ Leaderboard — calls render_leaderboard(load_uq_store())
64
  │ ├── MethodDetails.py 📘 Method configurations — per-method (selection-driven) technical
 
83
  │ ├── rl-td3/ RL-TD3-SHORT / RL-TD3 (TD3 variant of the RL experiments; RL-TD3 files are `rl-td3-long` in the NetCDF) results (per-benchmark files)
84
  │ ├── rl-ddpg/ RL-DDPG-SHORT / RL-DDPG (DDPG actor-critic; short RL step / standard-window RL step; RL-DDPG files are `rl-ddpg-long` in the NetCDF) results (per-benchmark files)
85
  │ ├── ces-eki-dmc_results/ CES-EKI-DMC ensemble-results files (UQ source)
86
+ │ ├── gboed-uq_results/ GBOED ensemble-results files (UQ source; 2026-10-16 GBOED-TMCMC/IES-TMCMC and 2026-10-19 IEKF-based variants, all four benchmarks; files are named `boed-*`: `tmcmc-eig` = GBOED-TMCMC, `iekf-eig` = GBOED-IEKF, `iekf` = IES-IEKF, `tmcmc` = IES-TMCMC; IES = Iterated Emulate-Sample)
87
  │ ├── gnki-uq_results/ IEKF (a.k.a GNKI) ensemble-results files (UQ source)
88
  │ └── abc/ ABC ensemble-results files (UQ source; per-benchmark files)
89
+ ├── offline-plots/ leaderboard_plot.py — matplotlib replica of the leaderboard charts
90
+ │ (failure rate, forward runs, iterations) for one benchmark/target,
91
+ │ serial vs ensemble panels, colors from METHOD_COLORS. Defaults to the
92
+ │ suitability-table target (1.1 opt / 1.5 UQ); writes to figures/.
93
+ │ `python offline-plots/leaderboard_plot.py [--board uq] [--methods ...]`
94
+ │ suitability_tables.py — regenerates tables/suitability_tables.tex (LaTeX
95
+ │ suitability grids, rows grouped by method family with \midrule) from the
96
+ │ current stores; `python offline-plots/suitability_tables.py`. New methods
97
+ │ must be listed in `SUITABILITY_FAMILIES` (method_registry.py) or it exits with an error.
98
+ ├── tables/ suitability_tables.tex (generated), method_taxonomy_tables.tex (hand-written)
99
  ├── examples/ lorenz_demo.ipynb + lorenz.py (reference, not app code)
100
  ├── From_Rob_6-10/ Raw collaborator data drop — NOT read by the app
101
  ├── .cache/ known_methods_snapshot.json (auto-generated, safe to delete)
 
216
  | `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | gradient (Kalman) | uq | after-optimize | — |
217
  | `adam` | ADAM | serial | gradient (exact) | optimization | none | — |
218
  | `lm` | LM | serial | gradient (exact) | optimization | none | gradient_descent |
219
+ | `gboed-tmcmc` | GBOED-TMCMC | parallel-interacting | general | uq | within-optimize | gboed, boed, tmcmc-eig, boed-tmcmc-eig, gboed-tmcmc-eig |
220
+ | `ies-iekf` | IES-IEKF | parallel-interacting | general | uq | within-optimize | boed-iekf |
221
+ | `gboed-iekf` | GBOED-IEKF | parallel-interacting | general | uq | within-optimize | gboed-iekf-eig, boed-iekf-eig |
222
+ | `ies-tmcmc` | IES-TMCMC | parallel-interacting | general | uq | within-optimize | boed-tmcmc |
223
  | `lm-kgmm` | LM-KGMM | serial | gradient (score) | optimization | none | — |
224
  | `lm-dsm` | LM-DSM | serial | gradient (score) | optimization | none | — |
225
  | `adam-kgmm` | ADAM-KGMM | serial | gradient (score) | optimization | none | — |
 
238
  2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
239
  keyed by **abbreviation** (e.g. `"TEKI"`). ⚠️ Metadata is split across both files.
240
  3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
241
+ 4. Add the abbreviation to a family in `SUITABILITY_FAMILIES` (`method_registry.py`; spelled-out family
242
+ names live in `FAMILY_FULL_NAMES` — add an entry there for any new family). Unlisted methods
243
+ appear in a trailing "Other" block of the online suitability table, and
244
+ `offline-plots/suitability_tables.py` refuses to run until they are assigned.
245
 
246
  ### Add an optimization benchmark dataset
247
  1. Add a `"BENCHMARK_NAME": [list_of_nc_paths]` entry to `DATASET_FILES` in `src/data_store.py`.
 
314
  - **Suitability table is fixed at `default_target`** — `_render_suitability_table()` always
315
  evaluates suitability at the `default_target` passed to `render_leaderboard()`, independent
316
  of what the user selects in the target radio control below it.
317
+ - **Family display names** — `SUITABILITY_FAMILIES` keys are short ids (`RL`, `GFDT`, ...); the UI shows `family_full_name(key)` from `FAMILY_FULL_NAMES` (suitability table, the family key under the home-page method table, and the per-family rows of the "Methods to display in charts" pills on both leaderboards; session keys `<prefix>_selected_methods__<family>`, with `<prefix>_selected_methods` mirroring the union).
318
+ - **Suitability table families** — rows of the suitability table are grouped by `SUITABILITY_FAMILIES`
319
+ (family order = dict order, row order = list order). Online (`_render_suitability_table`) this shows as a
320
+ leading "Family" column labelled on the first row of each block with alternating gray shading, since
321
+ `st.dataframe` cannot draw rules; the LaTeX tables use `\midrule` between blocks. A family only appears on a
322
+ board where one of its methods has data.
323
  - **Suitability criteria** — a method is green for a benchmark if there exists an ensemble size
324
  with `failure_rate < 20%` and `mean_budget ≤ 3× global_best` at `default_target`. Gray means
325
  no data for that benchmark; the method may be suitable on other benchmarks.
data/{gboed-uq_results/boed_l63_ensemble_results_2026-07-30_minimal.nc → adam_results/leaderboard_adam_l96_flux-force_2026-10-21.nc} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:52616c824f068f6de1eb9ccf67f56a1000b56d1cd54c911daa817df4b600e26b
3
- size 673057
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:539eb03a55d4ffe0a51b3b45db3b5dc0d0e159c8132e71639ba2eb5c0f72af95
3
+ size 11440
data/gboed-uq_results/{boed_l96_const-force_2026-07-30_minimal.nc → boed-iekf-eig_l63_ensemble_results_2026-10-19_minimal.nc} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:84062bc4afda563eba840efcf06c70143037660e99f258d6ecba84538a308b78
3
- size 809857
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f65ab0ae11bc8774dff077565a96dea07329cc1fe9c1055e01de4942bbb609f2
3
+ size 728387
data/gboed-uq_results/{boed_l96_vec-force_2026-08-07_minimal.nc → boed-iekf-eig_l96_const-force_2026-10-19_minimal.nc} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d1a3cd33db2cef7d81e31cf0a679e93955f3c851153eefbf2c719061b60330b9
3
- size 618337
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04951c1aa8553181100c69f4819c3df54d72a39c4d614624a56d778cef85c710
3
+ size 728387
data/gboed-uq_results/boed-iekf-eig_l96_flux-force_2026-10-19_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d65fc5289eece6735841ad4024b2fa3c9ec3d09e4a6c0442e3ef217399a9941
3
+ size 728387
data/gboed-uq_results/boed-iekf-eig_l96_vec-force_2026-10-19_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af1d2f624baf2e6c398718fa593caabccf7db232a133041bd3ed46eef28266a5
3
+ size 728387
data/gboed-uq_results/boed-iekf_l63_ensemble_results_2026-10-19_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a39050677da137a8cb4ab906c5e4d6f532c0a4ef9c7c7dd0c60885e988554d48
3
+ size 728383
data/gboed-uq_results/boed-iekf_l96_const-force_2026-10-19_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2fdf0355c4c2461ff96c1bb610a472fa9c4f5379b55a748f74eb089daf5e2be
3
+ size 728383
data/gboed-uq_results/boed-iekf_l96_flux-force_2026-10-19_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a61dab8820a1d681da06758a13ade609b2feeb1d0b9b95b645d858dd504d368d
3
+ size 728383
data/gboed-uq_results/boed-iekf_l96_vec-force_2026-10-19_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2779b4eec62487f8794e472fcf7abc3d2721549ce7d857c82f9ced4075551d7
3
+ size 728383
data/gboed-uq_results/boed-tmcmc-eig_l63_ensemble_results_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:add03f4917aeff93b36b051520b1341363c494905df5352c2ecdafa46d8be19a
3
+ size 728388
data/gboed-uq_results/boed-tmcmc-eig_l96_const-force_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2312bf2553c4fb38c47f4ab50eb0f2d339d9af21dbcdb08ec549d224eaeee79
3
+ size 728388
data/gboed-uq_results/boed-tmcmc-eig_l96_flux-force_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ea5592248a9ef9f739d95825cdfe9a01fd76c6c6c50ecf2e341139e26af4d4a
3
+ size 728388
data/gboed-uq_results/boed-tmcmc-eig_l96_vec-force_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5c7924216ef9add1bd56e294d7bded10c7b0ca2df696b9c3f2ceec66bc2e8a9
3
+ size 728388
data/gboed-uq_results/boed-tmcmc_l63_ensemble_results_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ed95ad74202e40bcf6b412f37d2f35b4d8b5a66f1ae9effee9dac19f7068aaa
3
+ size 728384
data/gboed-uq_results/boed-tmcmc_l96_const-force_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5072b6ec954baeb5add8097a29a33ca82e4d55a9f81965d43ee2bb629e5e644
3
+ size 728384
data/gboed-uq_results/boed-tmcmc_l96_flux-force_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cd0f8903089ebdb468e5b3e41ce08c89207905ef1f47135f1e5369fa1632203
3
+ size 728384
data/gboed-uq_results/boed-tmcmc_l96_vec-force_2026-10-16_minimal.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ac1182e23750b0db31239f52c8279771c8d6cbcfe66743184d2ffaa15b30194
3
+ size 728384
data/levenberg_marquardt_results/leaderboard_lm_l96_flux-force_2026-10-20.nc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e2a216e92286052411116fc527cb1d49e66272e522f48aeaf1d985aa3495976
3
+ size 11440
src/common/leaderboard.py CHANGED
@@ -25,13 +25,16 @@ import pandas as pd
25
  import streamlit as st
26
 
27
  try:
28
- from common.method_registry import METHOD_COLORS
29
  except ModuleNotFoundError:
30
- from src.common.method_registry import METHOD_COLORS
31
 
32
  _SUITABLE = "#009E73" # Okabe-Ito teal-green (colorblind-safe)
33
  _UNSUITABLE = "#C0392B" # dark red
34
  _UNTESTED = "#BDBDBD" # gray
 
 
 
35
 
36
 
37
  def _method_color(present_abbrevs: list[str]) -> alt.Color:
@@ -67,6 +70,24 @@ _UPDATE_TYPE_ORDER: dict[str, int] = {
67
  _RELIABLE_FAILURE_THRESHOLD = 20.0
68
 
69
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  def _render_suitability_table(
71
  store: pd.DataFrame,
72
  target_col: str,
@@ -80,16 +101,20 @@ def _render_suitability_table(
80
  store["benchmark"].unique().tolist(),
81
  key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
82
  )
83
- _abbr_update_type = (
84
- store[["abbreviation", "update_type"]].dropna()
85
- .drop_duplicates("abbreviation")
86
- .set_index("abbreviation")["update_type"]
87
- .to_dict()
88
- )
89
  methods = sorted(
90
  store["abbreviation"].dropna().unique().tolist(),
91
- key=lambda a: (_UPDATE_TYPE_ORDER.get(_abbr_update_type.get(a, ""), 99), a),
 
 
 
 
92
  )
 
93
 
94
  target_str = str(float(suitability_target))
95
  target_str_col = f"{target_col}_str"
@@ -144,13 +169,27 @@ def _render_suitability_table(
144
  display_df = pd.DataFrame(cell_text).T.rename(columns=col_labels)
145
  color_df = pd.DataFrame(cell_color).T.rename(columns=col_labels)
146
  display_df.index.name = "Method"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
 
148
  def _style(df: pd.DataFrame) -> pd.DataFrame:
149
  result = pd.DataFrame("", index=df.index, columns=df.columns)
150
  for row in df.index:
151
  for col in df.columns:
152
  bg = color_df.loc[row, col]
153
- fg = "#212529" if bg == _UNTESTED else "white"
154
  result.loc[row, col] = (
155
  f"background-color: {bg}; color: {fg}; "
156
  "text-align: center; font-weight: bold"
@@ -165,7 +204,8 @@ def _render_suitability_table(
165
  "**Red** (failed): data present but no ensemble size met the criteria; "
166
  "the percentage is the lowest failure rate (share of runs that did not reach the target) "
167
  "across ensemble sizes. "
168
- "**Gray (—)**: no data for this benchmark."
 
169
  )
170
  st.dataframe(
171
  display_df.style.apply(_style, axis=None),
@@ -184,7 +224,6 @@ def render_leaderboard(
184
  default_target: float,
185
  raw_page: str | None = None,
186
  show_failure_panel: bool = False,
187
- show_scoring_modes: bool = True,
188
  canonical_target_levels: list[float] | None = None,
189
  budget_store: pd.DataFrame | None = None,
190
  benchmark_dims: dict[str, tuple[int, int, int]] | None = None,
@@ -220,8 +259,6 @@ def render_leaderboard(
220
  show_failure_panel:
221
  If ``True``, render a grouped-bar failure-rate chart below the main
222
  performance chart.
223
- show_scoring_modes:
224
- If ``True``, show scoring-mode radio controls.
225
  canonical_target_levels:
226
  If provided, the target-level selector always offers exactly these
227
  values (as strings) regardless of what is present in the data. Use
@@ -263,18 +300,9 @@ def render_leaderboard(
263
  metric_store[target_col].astype(str).unique().tolist()
264
  )
265
 
266
- scoring_options = [
267
- "Mean Forward Model Runs",
268
- "Minimum Forward Model Runs",
269
- "Smallest Optimal Ensemble Size",
270
- "Custom Blend",
271
- ]
272
-
273
  # Session-state keys namespaced by state_prefix so two leaderboard pages
274
  # don't share control state.
275
  k_target = f"{state_prefix}_selected_target"
276
- k_scoring = f"{state_prefix}_scoring_mode"
277
- k_weight = f"{state_prefix}_fwdruns_weight_percent"
278
  k_methods = f"{state_prefix}_selected_methods"
279
 
280
  default_target_str = str(float(default_target))
@@ -286,17 +314,6 @@ def render_leaderboard(
286
  )
287
  selected_target = st.session_state[k_target]
288
 
289
- current_scoring_mode = st.session_state.get(k_scoring, "Mean Forward Model Runs")
290
- if current_scoring_mode not in scoring_options:
291
- current_scoring_mode = scoring_options[0]
292
-
293
- current_fwdruns_weight_percent = int(st.session_state.get(k_weight, 80))
294
- current_fwdruns_weight_percent = max(0, min(100, current_fwdruns_weight_percent))
295
-
296
- scoring_mode = current_scoring_mode
297
- fwdruns_weight = current_fwdruns_weight_percent / 100.0
298
- ensemble_weight = 1.0 - fwdruns_weight
299
-
300
  def build_scored_table(input_df: pd.DataFrame, add_rank: bool = True) -> pd.DataFrame:
301
  ranking_source = input_df[input_df[target_str_col] == selected_target]
302
  if ranking_source.empty:
@@ -316,7 +333,6 @@ def render_leaderboard(
316
  ["algorithm_type", "abbreviation", "Method"], as_index=False
317
  ).agg(
318
  **{"Mean Forward Model Runs": ("metric", "mean")},
319
- **{"Minimum Forward Model Runs": ("metric", "min")},
320
  **{"Ensemble Sizes Used": ("ensemble_size", "nunique")},
321
  )
322
 
@@ -351,7 +367,6 @@ def render_leaderboard(
351
 
352
  scored_df["Optimal Ensemble Size"] = scored_df["Optimal Ensemble Size"].round(2)
353
  scored_df["Mean Forward Model Runs"] = scored_df["Mean Forward Model Runs"].round(4)
354
- scored_df["Minimum Forward Model Runs"] = scored_df["Minimum Forward Model Runs"].round(4)
355
  scored_df["Mean Failure Rate (%)"] = scored_df["Mean Failure Rate (%)"].round(1)
356
 
357
  mean_runs_min = scored_df["Mean Forward Model Runs"].min()
@@ -363,47 +378,13 @@ def render_leaderboard(
363
  else:
364
  scored_df["mean_runs_score"] = 100.0
365
 
366
- minimum_runs_min = scored_df["Minimum Forward Model Runs"].min()
367
- minimum_runs_max = scored_df["Minimum Forward Model Runs"].max()
368
- if minimum_runs_max > minimum_runs_min:
369
- scored_df["minimum_runs_score"] = (
370
- 100.0 * (minimum_runs_max - scored_df["Minimum Forward Model Runs"]) / (minimum_runs_max - minimum_runs_min)
371
- )
372
- else:
373
- scored_df["minimum_runs_score"] = 100.0
374
-
375
- ens_min = scored_df["Optimal Ensemble Size"].min()
376
- ens_max = scored_df["Optimal Ensemble Size"].max()
377
- if ens_max > ens_min:
378
- scored_df["ensemble_score"] = (
379
- 100.0 * (ens_max - scored_df["Optimal Ensemble Size"]) / (ens_max - ens_min)
380
- )
381
- else:
382
- scored_df["ensemble_score"] = 100.0
383
-
384
  scored_df["_unreliable"] = (
385
  scored_df["Mean Failure Rate (%)"].fillna(0.0) >= _RELIABLE_FAILURE_THRESHOLD
386
  )
387
 
388
- if scoring_mode == "Mean Forward Model Runs":
389
- scored_df["Score"] = scored_df["mean_runs_score"]
390
- sort_columns = ["_unreliable", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
391
- ascending = [True, True, True, True]
392
- elif scoring_mode == "Minimum Forward Model Runs":
393
- scored_df["Score"] = scored_df["minimum_runs_score"]
394
- sort_columns = ["_unreliable", "Minimum Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
395
- ascending = [True, True, True, True]
396
- elif scoring_mode == "Smallest Optimal Ensemble Size":
397
- scored_df["Score"] = scored_df["ensemble_score"]
398
- sort_columns = ["_unreliable", "Optimal Ensemble Size", "Mean Forward Model Runs", "abbreviation"]
399
- ascending = [True, True, True, True]
400
- else:
401
- scored_df["Score"] = (
402
- fwdruns_weight * scored_df["mean_runs_score"]
403
- + ensemble_weight * scored_df["ensemble_score"]
404
- )
405
- sort_columns = ["_unreliable", "Score", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
406
- ascending = [True, False, True, True, True]
407
 
408
  scored_df = (
409
  scored_df.sort_values(sort_columns, ascending=ascending)
@@ -426,8 +407,8 @@ def render_leaderboard(
426
  if dnf_abbrevs:
427
  dnf_rows = failure_agg[failure_agg["abbreviation"].isin(dnf_abbrevs)].copy()
428
  for col in [
429
- "Score", "mean_runs_score", "minimum_runs_score", "ensemble_score",
430
- "Mean Forward Model Runs", "Minimum Forward Model Runs",
431
  "Optimal Ensemble Size", "Ensemble Sizes Used",
432
  ]:
433
  dnf_rows[col] = float("nan")
@@ -466,9 +447,22 @@ def render_leaderboard(
466
  # All runs failed at this target (or no data) — show every method so the
467
  # all-DNF warning/chart-suppression path still has something to work with.
468
  default_methods = available_methods
469
- saved_methods = st.session_state.get(k_methods, default_methods)
470
- valid_saved = [m for m in saved_methods if m in available_methods]
 
 
 
 
 
 
 
 
 
471
  st.session_state[k_methods] = valid_saved if valid_saved else default_methods
 
 
 
 
472
 
473
  # Attach the four taxonomy tags (constant per abbreviation, so this is a
474
  # plain lookup merge rather than a groupby key threaded through every
@@ -490,22 +484,12 @@ def render_leaderboard(
490
  na_position="last",
491
  ).drop(columns=["_update_type_sort", "_is_dnf"]).reset_index(drop=True)
492
 
493
- if scoring_mode == "Mean Forward Model Runs":
494
- score_basis = "mean forward-model runs at the selected target level (lower is better)"
495
- elif scoring_mode == "Minimum Forward Model Runs":
496
- score_basis = "minimum forward-model runs at the selected target level (lower is better)"
497
- elif scoring_mode == "Smallest Optimal Ensemble Size":
498
- score_basis = "mean optimal ensemble size at the selected target level (lower is better)"
499
- else:
500
- score_basis = (
501
- f"weighted blend of normalized forward-model-runs score ({fwdruns_weight:.0%}) "
502
- f"and normalized ensemble-size score ({ensemble_weight:.0%})"
503
- )
504
 
505
  # Controls — always visible (not collapsed behind a dropdown) so users can
506
- # change target/scoring/methods even when the current selection yields all
507
  # failures.
508
- st.subheader("Scoring & Target Controls")
509
  st.radio(
510
  target_label,
511
  options=target_options,
@@ -513,36 +497,20 @@ def render_leaderboard(
513
  key=k_target,
514
  )
515
 
516
- if show_scoring_modes:
517
- st.radio(
518
- "Scoring Method",
519
- options=scoring_options,
520
- horizontal=True,
521
- key=k_scoring,
 
 
522
  )
523
 
524
- if st.session_state.get(k_scoring, "Mean Forward Model Runs") == "Custom Blend":
525
- st.slider(
526
- "Blend Weight: Forward Runs vs Ensemble Size",
527
- min_value=0,
528
- max_value=100,
529
- step=5,
530
- key=k_weight,
531
- help=(
532
- "Higher forward-runs weight prioritizes fewer model evaluations; "
533
- "higher ensemble-size weight prioritizes smaller ensembles."
534
- ),
535
- )
536
-
537
- st.pills(
538
- "Methods to display in charts",
539
- options=available_methods,
540
- selection_mode="multi",
541
- key=k_methods,
542
- help="Defaults to the top 5 methods by current rank — click to add or remove.",
543
- )
544
-
545
- selected_methods = st.session_state.get(k_methods, available_methods)
546
  if not selected_methods:
547
  selected_methods = available_methods
548
 
@@ -840,7 +808,6 @@ def render_leaderboard(
840
  "emulator_use",
841
  "Score",
842
  "Mean Forward Model Runs",
843
- "Minimum Forward Model Runs",
844
  "Mean Failure Rate (%)",
845
  "Optimal Ensemble Size",
846
  "Ensemble Sizes Used",
@@ -868,7 +835,6 @@ def render_leaderboard(
868
  "Method": st.column_config.TextColumn("Method"),
869
  "abbreviation": st.column_config.TextColumn("Abbrev."),
870
  "Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
871
- "Minimum Forward Model Runs": st.column_config.NumberColumn("Minimum Forward Model Runs", format="%.4f"),
872
  "Score": st.column_config.ProgressColumn("Score (0-100)", min_value=0.0, max_value=100.0, format="%.1f"),
873
  "Optimal Ensemble Size": st.column_config.NumberColumn("Mean Optimal Ensemble Size", format="%.2f"),
874
  "Mean Failure Rate (%)": st.column_config.NumberColumn("Mean Failure Rate (%)", format="%.1f"),
 
25
  import streamlit as st
26
 
27
  try:
28
+ from common.method_registry import METHOD_COLORS, SUITABILITY_FAMILIES, family_full_name
29
  except ModuleNotFoundError:
30
+ from src.common.method_registry import METHOD_COLORS, SUITABILITY_FAMILIES, family_full_name
31
 
32
  _SUITABLE = "#009E73" # Okabe-Ito teal-green (colorblind-safe)
33
  _UNSUITABLE = "#C0392B" # dark red
34
  _UNTESTED = "#BDBDBD" # gray
35
+ # Alternating light grays for the family-label column of the suitability table, so
36
+ # consecutive family blocks are visually distinct.
37
+ _FAMILY_BGS = ("#E9ECEF", "#CED4DA")
38
 
39
 
40
  def _method_color(present_abbrevs: list[str]) -> alt.Color:
 
70
  _RELIABLE_FAILURE_THRESHOLD = 20.0
71
 
72
 
73
+ def _group_by_family(methods: list[str]) -> list[tuple[str, list[str]]]:
74
+ """Group method abbreviations into (family key, methods) in SUITABILITY_FAMILIES order.
75
+
76
+ Methods not listed in any family go into a trailing "Other" group.
77
+ """
78
+ groups: list[tuple[str, list[str]]] = []
79
+ placed: set[str] = set()
80
+ for fam, members in SUITABILITY_FAMILIES.items():
81
+ present = [m for m in members if m in methods]
82
+ if present:
83
+ groups.append((fam, present))
84
+ placed.update(present)
85
+ rest = [m for m in methods if m not in placed]
86
+ if rest:
87
+ groups.append(("Other", sorted(rest)))
88
+ return groups
89
+
90
+
91
  def _render_suitability_table(
92
  store: pd.DataFrame,
93
  target_col: str,
 
101
  store["benchmark"].unique().tolist(),
102
  key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
103
  )
104
+ # Rows are grouped into method families (SUITABILITY_FAMILIES); methods not listed
105
+ # there fall into a trailing "Other" family.
106
+ _family_of = {m: fam for fam, ms in SUITABILITY_FAMILIES.items() for m in ms}
107
+ _family_rank = {fam: i for i, fam in enumerate(SUITABILITY_FAMILIES)}
108
+ _row_rank = {m: i for ms in SUITABILITY_FAMILIES.values() for i, m in enumerate(ms)}
 
109
  methods = sorted(
110
  store["abbreviation"].dropna().unique().tolist(),
111
+ key=lambda a: (
112
+ _family_rank.get(_family_of.get(a, ""), len(_family_rank)),
113
+ _row_rank.get(a, 0),
114
+ a,
115
+ ),
116
  )
117
+ family_label = {m: family_full_name(_family_of.get(m, "Other")) for m in methods}
118
 
119
  target_str = str(float(suitability_target))
120
  target_str_col = f"{target_col}_str"
 
169
  display_df = pd.DataFrame(cell_text).T.rename(columns=col_labels)
170
  color_df = pd.DataFrame(cell_color).T.rename(columns=col_labels)
171
  display_df.index.name = "Method"
172
+ # Leading "Family" column, labelled only on the first row of each family block so
173
+ # the families read as blocks (st.dataframe cannot draw rules between row groups).
174
+ shown_family = [
175
+ fam if i == 0 or fam != family_label[methods[i - 1]] else ""
176
+ for i, fam in enumerate(family_label[m] for m in methods)
177
+ ]
178
+ display_df.insert(0, "Family", shown_family)
179
+ block_idx, prev = -1, None
180
+ family_bg = []
181
+ for m in methods:
182
+ if family_label[m] != prev:
183
+ block_idx, prev = block_idx + 1, family_label[m]
184
+ family_bg.append(_FAMILY_BGS[block_idx % 2])
185
+ color_df.insert(0, "Family", family_bg)
186
 
187
  def _style(df: pd.DataFrame) -> pd.DataFrame:
188
  result = pd.DataFrame("", index=df.index, columns=df.columns)
189
  for row in df.index:
190
  for col in df.columns:
191
  bg = color_df.loc[row, col]
192
+ fg = "#212529" if bg == _UNTESTED or bg in _FAMILY_BGS else "white"
193
  result.loc[row, col] = (
194
  f"background-color: {bg}; color: {fg}; "
195
  "text-align: center; font-weight: bold"
 
204
  "**Red** (failed): data present but no ensemble size met the criteria; "
205
  "the percentage is the lowest failure rate (share of runs that did not reach the target) "
206
  "across ensemble sizes. "
207
+ "**Gray (—)**: no data for this benchmark. "
208
+ "Rows are grouped into method families of similar construction (left column)."
209
  )
210
  st.dataframe(
211
  display_df.style.apply(_style, axis=None),
 
224
  default_target: float,
225
  raw_page: str | None = None,
226
  show_failure_panel: bool = False,
 
227
  canonical_target_levels: list[float] | None = None,
228
  budget_store: pd.DataFrame | None = None,
229
  benchmark_dims: dict[str, tuple[int, int, int]] | None = None,
 
259
  show_failure_panel:
260
  If ``True``, render a grouped-bar failure-rate chart below the main
261
  performance chart.
 
 
262
  canonical_target_levels:
263
  If provided, the target-level selector always offers exactly these
264
  values (as strings) regardless of what is present in the data. Use
 
300
  metric_store[target_col].astype(str).unique().tolist()
301
  )
302
 
 
 
 
 
 
 
 
303
  # Session-state keys namespaced by state_prefix so two leaderboard pages
304
  # don't share control state.
305
  k_target = f"{state_prefix}_selected_target"
 
 
306
  k_methods = f"{state_prefix}_selected_methods"
307
 
308
  default_target_str = str(float(default_target))
 
314
  )
315
  selected_target = st.session_state[k_target]
316
 
 
 
 
 
 
 
 
 
 
 
 
317
  def build_scored_table(input_df: pd.DataFrame, add_rank: bool = True) -> pd.DataFrame:
318
  ranking_source = input_df[input_df[target_str_col] == selected_target]
319
  if ranking_source.empty:
 
333
  ["algorithm_type", "abbreviation", "Method"], as_index=False
334
  ).agg(
335
  **{"Mean Forward Model Runs": ("metric", "mean")},
 
336
  **{"Ensemble Sizes Used": ("ensemble_size", "nunique")},
337
  )
338
 
 
367
 
368
  scored_df["Optimal Ensemble Size"] = scored_df["Optimal Ensemble Size"].round(2)
369
  scored_df["Mean Forward Model Runs"] = scored_df["Mean Forward Model Runs"].round(4)
 
370
  scored_df["Mean Failure Rate (%)"] = scored_df["Mean Failure Rate (%)"].round(1)
371
 
372
  mean_runs_min = scored_df["Mean Forward Model Runs"].min()
 
378
  else:
379
  scored_df["mean_runs_score"] = 100.0
380
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
381
  scored_df["_unreliable"] = (
382
  scored_df["Mean Failure Rate (%)"].fillna(0.0) >= _RELIABLE_FAILURE_THRESHOLD
383
  )
384
 
385
+ scored_df["Score"] = scored_df["mean_runs_score"]
386
+ sort_columns = ["_unreliable", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
387
+ ascending = [True, True, True, True]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
388
 
389
  scored_df = (
390
  scored_df.sort_values(sort_columns, ascending=ascending)
 
407
  if dnf_abbrevs:
408
  dnf_rows = failure_agg[failure_agg["abbreviation"].isin(dnf_abbrevs)].copy()
409
  for col in [
410
+ "Score", "mean_runs_score",
411
+ "Mean Forward Model Runs",
412
  "Optimal Ensemble Size", "Ensemble Sizes Used",
413
  ]:
414
  dnf_rows[col] = float("nan")
 
447
  # All runs failed at this target (or no data) — show every method so the
448
  # all-DNF warning/chart-suppression path still has something to work with.
449
  default_methods = available_methods
450
+ # The selector is one pills widget per method family (one row each); the
451
+ # selection is the union of the family widgets, mirrored into k_methods.
452
+ method_groups = _group_by_family(available_methods)
453
+ family_keys = {fam: f"{k_methods}__{fam}" for fam, _ in method_groups}
454
+ if any(wkey in st.session_state for wkey in family_keys.values()):
455
+ saved_methods = [
456
+ m for wkey in family_keys.values() for m in st.session_state.get(wkey) or []
457
+ ]
458
+ else:
459
+ saved_methods = default_methods
460
+ valid_saved = [m for m in saved_methods if m in available_methods]
461
  st.session_state[k_methods] = valid_saved if valid_saved else default_methods
462
+ for fam, members in method_groups:
463
+ st.session_state[family_keys[fam]] = [
464
+ m for m in st.session_state[k_methods] if m in members
465
+ ]
466
 
467
  # Attach the four taxonomy tags (constant per abbreviation, so this is a
468
  # plain lookup merge rather than a groupby key threaded through every
 
484
  na_position="last",
485
  ).drop(columns=["_update_type_sort", "_is_dnf"]).reset_index(drop=True)
486
 
487
+ score_basis = "mean forward-model runs at the selected target level (lower is better)"
 
 
 
 
 
 
 
 
 
 
488
 
489
  # Controls — always visible (not collapsed behind a dropdown) so users can
490
+ # change target/methods even when the current selection yields all
491
  # failures.
492
+ st.subheader("Target & Method Controls")
493
  st.radio(
494
  target_label,
495
  options=target_options,
 
497
  key=k_target,
498
  )
499
 
500
+ st.markdown("**Methods to display in charts**")
501
+ st.caption("Defaults to the top 5 methods by current rank — click to add or remove. Rows are method families.")
502
+ for fam, members in method_groups:
503
+ st.pills(
504
+ family_full_name(fam),
505
+ options=members,
506
+ selection_mode="multi",
507
+ key=family_keys[fam],
508
  )
509
 
510
+ selected_methods = [
511
+ m for fam, _ in method_groups for m in st.session_state.get(family_keys[fam]) or []
512
+ ]
513
+ st.session_state[k_methods] = selected_methods
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
514
  if not selected_methods:
515
  selected_methods = available_methods
516
 
 
808
  "emulator_use",
809
  "Score",
810
  "Mean Forward Model Runs",
 
811
  "Mean Failure Rate (%)",
812
  "Optimal Ensemble Size",
813
  "Ensemble Sizes Used",
 
835
  "Method": st.column_config.TextColumn("Method"),
836
  "abbreviation": st.column_config.TextColumn("Abbrev."),
837
  "Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
 
838
  "Score": st.column_config.ProgressColumn("Score (0-100)", min_value=0.0, max_value=100.0, format="%.1f"),
839
  "Optimal Ensemble Size": st.column_config.NumberColumn("Mean Optimal Ensemble Size", format="%.2f"),
840
  "Mean Failure Rate (%)": st.column_config.NumberColumn("Mean Failure Rate (%)", format="%.1f"),
src/common/method_registry.py CHANGED
@@ -140,14 +140,14 @@ KNOWN_METHODS = {
140
  "emulator_use": "none",
141
  "aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
142
  },
143
- "gboed": {
144
- "abbreviation": "GBOED",
145
- "Method": "Goal-oriented Bayesian Optimal Experimental Design",
146
  "parallelism": "parallel-interacting",
147
  "update_type": "general",
148
  "method_goal": "uq",
149
  "emulator_use": "within-optimize",
150
- "aliases": ["gboed", "boed"],
151
  },
152
  "lm-kgmm": {
153
  "abbreviation": "LM-KGMM",
@@ -237,6 +237,33 @@ KNOWN_METHODS = {
237
  "emulator_use": "none",
238
  "aliases": ["rl-td3", "rl-td3-long"],
239
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
240
  }
241
 
242
 
@@ -292,6 +319,41 @@ METHOD_COLORS: dict[str, str] = {
292
  }
293
 
294
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
295
  def build_alias_lookup() -> dict[str, str]:
296
  lookup: dict[str, str] = {}
297
  for canonical_name, meta in KNOWN_METHODS.items():
 
140
  "emulator_use": "none",
141
  "aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
142
  },
143
+ "gboed-tmcmc": {
144
+ "abbreviation": "GBOED-TMCMC",
145
+ "Method": "Goal-oriented Bayesian Optimal Experimental Design (TMCMC)",
146
  "parallelism": "parallel-interacting",
147
  "update_type": "general",
148
  "method_goal": "uq",
149
  "emulator_use": "within-optimize",
150
+ "aliases": ["gboed", "gboed-tmcmc", "boed", "gboed-tmcmc-eig", "boed-tmcmc-eig", "tmcmc-eig"],
151
  },
152
  "lm-kgmm": {
153
  "abbreviation": "LM-KGMM",
 
237
  "emulator_use": "none",
238
  "aliases": ["rl-td3", "rl-td3-long"],
239
  },
240
+ "ies-iekf": {
241
+ "abbreviation": "IES-IEKF",
242
+ "Method": "Iterated Emulate-Sample (IEKF)",
243
+ "parallelism": "parallel-interacting",
244
+ "update_type": "general",
245
+ "method_goal": "uq",
246
+ "emulator_use": "within-optimize",
247
+ "aliases": ["ies-iekf", "boed-iekf"],
248
+ },
249
+ "gboed-iekf": {
250
+ "abbreviation": "GBOED-IEKF",
251
+ "Method": "Goal-oriented Bayesian Optimal Experimental Design (IEKF)",
252
+ "parallelism": "parallel-interacting",
253
+ "update_type": "general",
254
+ "method_goal": "uq",
255
+ "emulator_use": "within-optimize",
256
+ "aliases": ["gboed-iekf", "gboed-iekf-eig", "boed-iekf-eig"],
257
+ },
258
+ "ies-tmcmc": {
259
+ "abbreviation": "IES-TMCMC",
260
+ "Method": "Iterated Emulate-Sample (TMCMC)",
261
+ "parallelism": "parallel-interacting",
262
+ "update_type": "general",
263
+ "method_goal": "uq",
264
+ "emulator_use": "within-optimize",
265
+ "aliases": ["ies-tmcmc", "boed-tmcmc"],
266
+ },
267
  }
268
 
269
 
 
319
  }
320
 
321
 
322
+ # Method families used to group rows in the suitability tables (the online table in
323
+ # common/leaderboard.py and the LaTeX tables from offline-plots/suitability_tables.py).
324
+ # The idea is to put methods of similar construction — and so similar performance —
325
+ # next to each other. Dict order = family order; list order = row order within a
326
+ # family. A family only appears on a board if one of its methods has data there.
327
+ # Every method that can appear in a suitability table must be listed here.
328
+ SUITABILITY_FAMILIES: dict[str, list[str]] = {
329
+ "CES": ["CES-EKI-CONST", "CES-EKI-DMC", "CES-IEKF-CONST"],
330
+ "Kalman": ["ETKI", "IEKF", "TEKI", "UKI"],
331
+ "Gradient": ["LM", "ADAM"],
332
+ "GFDT": ["LM-DSM", "ADAM-DSM", "LM-KGMM", "ADAM-KGMM"],
333
+ "RL": ["RL-DDPG", "RL-TD3", "RL-DDPG-SHORT", "RL-TD3-SHORT"],
334
+ "IES": ["IES-IEKF", "IES-TMCMC"],
335
+ "GBOED": ["GBOED-IEKF", "GBOED-TMCMC"],
336
+ "General": ["ABC", "HM"],
337
+ }
338
+
339
+ # Spelled-out display names for the SUITABILITY_FAMILIES keys (suitability table,
340
+ # home-page key, and the grouped method selector on the leaderboards).
341
+ FAMILY_FULL_NAMES: dict[str, str] = {
342
+ "CES": "Calibrate, Emulate, Sample (CES)",
343
+ "Kalman": "Kalman",
344
+ "Gradient": "Gradient",
345
+ "GFDT": "Generalized Fluctuation-Dissipation (GFDT)",
346
+ "RL": "Reinforcement Learning (RL)",
347
+ "IES": "Iterated Emulate-Sample (IES)",
348
+ "GBOED": "Goal-oriented Bayesian Experimental Design (GBOED)",
349
+ "General": "General",
350
+ }
351
+
352
+
353
+ def family_full_name(family: str) -> str:
354
+ return FAMILY_FULL_NAMES.get(family, family)
355
+
356
+
357
  def build_alias_lookup() -> dict[str, str]:
358
  lookup: dict[str, str] = {}
359
  for canonical_name, meta in KNOWN_METHODS.items():
src/data_store.py CHANGED
@@ -65,6 +65,8 @@ DATASET_FILES = {
65
  "etki_results/leaderboard_etki_l96_flux-force_2026-09-10.nc",
66
  "iekf_results/leaderboard_iekf_l96_flux-force_2026-09-10.nc",
67
  "UKI_results/uki_l96_nn_forcing_ensemble_results.nc",
 
 
68
  "opt_kgmm_score_lm/leaderboard_sblm_kgmm_parallel_l96_flux-force_2026-09-02.nc",
69
  "opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l96_flux-force_2026-08-12.nc",
70
  "opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l96_flux-force_2026-09-11.nc",
@@ -129,7 +131,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
129
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l63_ensemble_results_2026-07-09_minimal.nc"),
130
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l63_ensemble_results_2026-07-17_minimal.nc"),
131
  ("hm", "history-matching-uq_results/history-matching_l63_ensemble_results_2026-09-15_minimal.nc"),
132
- ("gboed", "gboed-uq_results/boed_l63_ensemble_results_2026-07-30_minimal.nc"),
 
 
 
133
  ("abc", "abc/leaderboard_abc_l63_2026-09-10_minimal.nc"),
134
  ],
135
  "L96": [
@@ -138,7 +143,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
138
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_ensemble_results_2026-07-09_minimal.nc"),
139
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_ensemble_results_2026-07-17_minimal.nc"),
140
  ("hm", "history-matching-uq_results/history-matching_l96_const-force_2026-09-15_minimal.nc"),
141
- ("gboed", "gboed-uq_results/boed_l96_const-force_2026-07-30_minimal.nc"),
 
 
 
142
  ("abc", "abc/leaderboard_abc_l96_const-force_2026-09-10_minimal.nc"),
143
  ],
144
  "L96_NN_FORCING": [
@@ -147,6 +155,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
147
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_nn_forcing_ensemble_results_2026-07-09_minimal.nc"),
148
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_nn_forcing_ensemble_results_2026-07-17_minimal.nc"),
149
  ("hm", "history-matching-uq_results/history-matching_l96_flux-force_2026-09-15_minimal.nc"),
 
 
 
 
150
  ("abc", "abc/leaderboard_abc_l96_flux-force_2026-09-10_minimal.nc"),
151
  ],
152
  "L96_SPATIAL_FORCING": [
@@ -155,7 +167,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
155
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_spatial_forcing_ensemble_results_2026-07-09_minimal.nc"),
156
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_spatial_forcing_ensemble_results_2026-07-17_minimal.nc"),
157
  ("hm", "history-matching-uq_results/history-matching_l96_vec-force_2026-09-15_minimal.nc"),
158
- ("gboed", "gboed-uq_results/boed_l96_vec-force_2026-08-07_minimal.nc"),
 
 
 
159
  ("abc", "abc/leaderboard_abc_l96_vec-force_2026-09-10_minimal.nc"),
160
  ],
161
  }
 
65
  "etki_results/leaderboard_etki_l96_flux-force_2026-09-10.nc",
66
  "iekf_results/leaderboard_iekf_l96_flux-force_2026-09-10.nc",
67
  "UKI_results/uki_l96_nn_forcing_ensemble_results.nc",
68
+ "adam_results/leaderboard_adam_l96_flux-force_2026-10-21.nc",
69
+ "levenberg_marquardt_results/leaderboard_lm_l96_flux-force_2026-10-20.nc",
70
  "opt_kgmm_score_lm/leaderboard_sblm_kgmm_parallel_l96_flux-force_2026-09-02.nc",
71
  "opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l96_flux-force_2026-08-12.nc",
72
  "opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l96_flux-force_2026-09-11.nc",
 
131
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l63_ensemble_results_2026-07-09_minimal.nc"),
132
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l63_ensemble_results_2026-07-17_minimal.nc"),
133
  ("hm", "history-matching-uq_results/history-matching_l63_ensemble_results_2026-09-15_minimal.nc"),
134
+ ("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l63_ensemble_results_2026-10-16_minimal.nc"),
135
+ ("ies-iekf", "gboed-uq_results/boed-iekf_l63_ensemble_results_2026-10-19_minimal.nc"),
136
+ ("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l63_ensemble_results_2026-10-19_minimal.nc"),
137
+ ("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l63_ensemble_results_2026-10-16_minimal.nc"),
138
  ("abc", "abc/leaderboard_abc_l63_2026-09-10_minimal.nc"),
139
  ],
140
  "L96": [
 
143
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_ensemble_results_2026-07-09_minimal.nc"),
144
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_ensemble_results_2026-07-17_minimal.nc"),
145
  ("hm", "history-matching-uq_results/history-matching_l96_const-force_2026-09-15_minimal.nc"),
146
+ ("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l96_const-force_2026-10-16_minimal.nc"),
147
+ ("ies-iekf", "gboed-uq_results/boed-iekf_l96_const-force_2026-10-19_minimal.nc"),
148
+ ("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l96_const-force_2026-10-19_minimal.nc"),
149
+ ("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l96_const-force_2026-10-16_minimal.nc"),
150
  ("abc", "abc/leaderboard_abc_l96_const-force_2026-09-10_minimal.nc"),
151
  ],
152
  "L96_NN_FORCING": [
 
155
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_nn_forcing_ensemble_results_2026-07-09_minimal.nc"),
156
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_nn_forcing_ensemble_results_2026-07-17_minimal.nc"),
157
  ("hm", "history-matching-uq_results/history-matching_l96_flux-force_2026-09-15_minimal.nc"),
158
+ ("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l96_flux-force_2026-10-16_minimal.nc"),
159
+ ("ies-iekf", "gboed-uq_results/boed-iekf_l96_flux-force_2026-10-19_minimal.nc"),
160
+ ("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l96_flux-force_2026-10-19_minimal.nc"),
161
+ ("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l96_flux-force_2026-10-16_minimal.nc"),
162
  ("abc", "abc/leaderboard_abc_l96_flux-force_2026-09-10_minimal.nc"),
163
  ],
164
  "L96_SPATIAL_FORCING": [
 
167
  ("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_spatial_forcing_ensemble_results_2026-07-09_minimal.nc"),
168
  ("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_spatial_forcing_ensemble_results_2026-07-17_minimal.nc"),
169
  ("hm", "history-matching-uq_results/history-matching_l96_vec-force_2026-09-15_minimal.nc"),
170
+ ("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l96_vec-force_2026-10-16_minimal.nc"),
171
+ ("ies-iekf", "gboed-uq_results/boed-iekf_l96_vec-force_2026-10-19_minimal.nc"),
172
+ ("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l96_vec-force_2026-10-19_minimal.nc"),
173
+ ("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l96_vec-force_2026-10-16_minimal.nc"),
174
  ("abc", "abc/leaderboard_abc_l96_vec-force_2026-09-10_minimal.nc"),
175
  ],
176
  }
src/pages/MethodDetails.py CHANGED
@@ -307,7 +307,7 @@ method_meta = {
307
  ],
308
  "notes": "**Empirically also gets stuck in local minima near the initial condition, regardless of the damping schedule** — the same failure mode as ADAM, unsurprising for any single-trajectory local optimizer on this multimodal, chaotic-dynamics calibration surface.",
309
  },
310
- "GBOED": {
311
  "citation": "Holthuijzen et al. 2026",
312
  "url": "https://arxiv.org/abs/2508.13071",
313
  "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — Goal-oriented BOED, Algorithm 2 of Holthuijzen et al. 2026.",
@@ -330,12 +330,18 @@ method_meta = {
330
  "switches from greedy to joint optimization once a greedy step's cost exceeds a threshold."
331
  ),
332
  "params": [
 
 
 
 
 
333
  {"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
334
- {"Parameter": "eig_outer_iters", "Value": "20"},
335
  {"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
 
336
  {"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
337
  ],
338
- "notes": "**Empirically, the inner EIG-maximization loop over candidate batches is a frequent source of problems (hitting the call-limit / outer-iteration guardrails, i.e. \"under-optimized\" batches), and every iteration slows down as the cumulative GP training set grows** (GP fit/predict cost scales super-linearly in training-set size) — this is also why the repo's own config drops to a smaller `max_iters` on the higher-dimensional L96 cases (\"it's so slow, so let's start here\").",
339
  },
340
  "LM-KGMM": {
341
  "citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025",
@@ -622,6 +628,132 @@ method_meta = {
622
  "targets on L96 flux-force or vec-force within the budget."
623
  ),
624
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
625
  }
626
 
627
  # KNOWN_METHODS is the source of truth for the abbreviation/full-name list,
 
307
  ],
308
  "notes": "**Empirically also gets stuck in local minima near the initial condition, regardless of the damping schedule** — the same failure mode as ADAM, unsurprising for any single-trajectory local optimizer on this multimodal, chaotic-dynamics calibration surface.",
309
  },
310
+ "GBOED-TMCMC": {
311
  "citation": "Holthuijzen et al. 2026",
312
  "url": "https://arxiv.org/abs/2508.13071",
313
  "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — Goal-oriented BOED, Algorithm 2 of Holthuijzen et al. 2026.",
 
330
  "switches from greedy to joint optimization once a greedy step's cost exceeds a threshold."
331
  ),
332
  "params": [
333
+ {"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"},
334
+ {"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"},
335
+ {"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"},
336
+ {"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"},
337
+ {"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"},
338
  {"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
339
+ {"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"},
340
  {"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
341
+ {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
342
  {"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
343
  ],
344
+ "notes": "**Both the Transitional MCMC (TMCMC) posterior sampling and the EIG batch optimization contribute to performance.** TMCMC needs a sufficiently large population (`n_posterior_samples`) to represent the posterior, particularly in higher dimensions; its cost grows roughly linearly with population size, and `tmcmc_burnin`/`tmcmc_thin` multiply the per-stage cost. The EIG objective uses only a small posterior subsample (`n_eig_posterior_samples`), as EIG costs ~n³ per output mode. **The inner EIG-maximization loop is a frequent source of problems (hitting the call-limit / outer-iteration guardrails, i.e. \"under-optimized\" batches), and every iteration slows down as the cumulative GP training set grows**, which is why the higher-dimensional L96 cases use a smaller `max_iters`.",
345
  },
346
  "LM-KGMM": {
347
  "citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025",
 
628
  "targets on L96 flux-force or vec-force within the budget."
629
  ),
630
  },
631
+ "IES-IEKF": {
632
+ "citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025",
633
+ "url": "https://doi.org/10.1029/2024MS004713",
634
+ "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `acquire_batch`), variant `:iekf` (leaderboard key `boed-iekf`).",
635
+ "math": (
636
+ "An iterated emulate-sample loop. Iteration 1 forward-evaluates an "
637
+ "`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later "
638
+ "iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; "
639
+ "(ii) draws an approximate posterior by running an IEKF (`GaussNewtonInversion`) on the GP-mean "
640
+ "forward map — no true forward evaluations — starting from 1000 LHS prior draws; (iii) takes the next "
641
+ "`N_ens` forward-model evaluation points as a random subsample of that IEKF ensemble. The batch is "
642
+ "forward-evaluated and appended to the training set, so each iteration emulates the forward model, "
643
+ "samples the posterior from the emulator, and spends its forward runs where that posterior has mass."
644
+ ),
645
+ "guardrails": (
646
+ "The posterior sampler is a fixed-step IEKF (`DefaultScheduler(0.1)`, 30 steps, total pseudo-time "
647
+ "T = 3), so it is a finite-time approximate posterior rather than a converged one; the observation "
648
+ "noise in whitened output space is the identity and the GP predictive variance is ignored. GP fits "
649
+ "keep kernel and noise hyperparameters inside bounds set relative to each mode's data scale (±2 "
650
+ "decades on length scales and signal std; noise std between 1e-3 and 1 of the signal std) to avoid "
651
+ "degenerate near-zero-noise fits, with `Fminbox` capped at 50 inner / 3 outer iterations. Inputs "
652
+ "and outputs are PCA-whitened and truncated against the prior and the observation covariance."
653
+ ),
654
+ "params": [
655
+ {"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"},
656
+ {"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"},
657
+ {"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the IEKF ensemble)"},
658
+ {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
659
+ {"Parameter": "retain_var (output whitening)", "Value": "0.99"},
660
+ {"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"},
661
+ ],
662
+ "notes": (
663
+ "**Cheap per iteration:** no MCMC and no batch optimization, only GP-mean predictions "
664
+ "inside the IEKF. The batch is a plain draw from the posterior with no diversity criterion, so points are "
665
+ "not spread out deliberately. In the current UQ data (c = 1.5, best ensemble size) it needs a "
666
+ "mean budget of 5.0 on L63 and 5.4 on L96 const-force, but on the two high-dimensional L96 cases "
667
+ "(vec-force and NN flux-force) it fails on at least 35% and 40% of seeds, respectively, at every ensemble size."
668
+ ),
669
+ },
670
+ "GBOED-IEKF": {
671
+ "citation": "Holthuijzen et al. 2026 (EIG acquisition); IEKF posterior sampler replaces ST-MCMC",
672
+ "url": "https://arxiv.org/abs/2508.13071",
673
+ "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `optimize_batch`, `acquire_batch`), variant `:iekf_eig` (leaderboard key `boed-iekf-eig`).",
674
+ "math": (
675
+ "GBOED with the posterior sampler swapped for an IEKF. Iteration 1 forward-evaluates an `N_ens`-point "
676
+ "Latin-hypercube design in the truncated prior-whitened space. Each later iteration: (i) refits one GP "
677
+ "per whitened output mode on all `(θ, G(θ))` pairs so far; (ii) draws an approximate posterior by "
678
+ "running an IEKF (`GaussNewtonInversion`) on the GP-mean forward map — no true forward evaluations; "
679
+ "(iii) chooses the next `N_ens` evaluation points by jointly maximizing the closed-form Expected "
680
+ "Information Gain (EIG) of the whole batch against a 50-member random subsample X' of the IEKF "
681
+ "ensemble, starting from a random subsample of the full ensemble."
682
+ ),
683
+ "math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}",
684
+ "guardrails": (
685
+ "The IEKF posterior is a fixed-step, finite-time approximation (`DefaultScheduler(0.1)`, 30 steps, "
686
+ "T = 3) that ignores the GP predictive variance. The batch is optimized with `Optim.jl`'s "
687
+ "`LBFGS` inside the χ² ball of the prior-whitened space (radial `tanh` squash, so every candidate "
688
+ "has radius below √χ²_k(0.999)), with a hard cap on raw EIG evaluations (`eig_call_limit`) and an "
689
+ "outer-iteration cap (`eig_outer_iters`) enforced manually, logging a warning that the returned "
690
+ "batch may be \"under-optimized\" if either is hit; a diagonal jitter (`eig_jitter`, relative to the "
691
+ "GP signal variance) stabilizes the covariance solves and log-determinants. GP fits use the same "
692
+ "data-relative hyperparameter bounds and iteration caps as IES-IEKF."
693
+ ),
694
+ "params": [
695
+ {"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"},
696
+ {"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"},
697
+ {"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"},
698
+ {"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"},
699
+ {"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"},
700
+ {"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
701
+ {"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"},
702
+ {"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
703
+ {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
704
+ {"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
705
+ {"Parameter": "retain_var / retain_var_input", "Value": "0.99 / 0.99 (L63, L96 const), 0.9 (vec, flux)"},
706
+ ],
707
+ "notes": (
708
+ "**Adds the EIG batch optimization to IES-IEKF while keeping the cheap IEKF posterior**, so it avoids "
709
+ "ST-MCMC cost but still pays for the inner EIG-maximization loop, which costs ~n³ per output mode in "
710
+ "the subsample size and slows every iteration as the cumulative GP training set grows; hitting its "
711
+ "call or outer-iteration limits gives \"under-optimized\" batches. In the current UQ data (c = 1.5, "
712
+ "best ensemble size) it has the lowest mean budget of the four variants on L63 (4.8) and ties "
713
+ "GBOED-TMCMC on L96 const-force (4.4). On the two high-dimensional L96 cases it has low budgets "
714
+ "(55 on vec-force, 50 on NN flux-force) but fails on at least 35% and 40% of seeds, respectively, "
715
+ "at every ensemble size, so it is not suitable there under the 20% failure threshold."
716
+ ),
717
+ },
718
+ "IES-TMCMC": {
719
+ "citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025",
720
+ "url": "https://doi.org/10.1029/2024MS004713",
721
+ "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_tmcmc`, `boed_loglik`, `acquire_batch`), variant `:tmcmc` (leaderboard key `boed-tmcmc`).",
722
+ "math": (
723
+ "An iterated emulate-sample loop. Iteration 1 forward-evaluates an "
724
+ "`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later "
725
+ "iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; "
726
+ "(ii) draws posterior samples from the GP-based posterior via Transitional/Sequential MCMC "
727
+ "(Ching & Chen, 2007), using the GP-marginal Gaussian likelihood below; (iii) takes the next `N_ens` "
728
+ "forward-model evaluation points as a random subsample of those posterior draws. The batch is "
729
+ "forward-evaluated and appended to the training set, so each iteration emulates the forward model, "
730
+ "samples the posterior from the emulator, and spends its forward runs where that posterior has mass."
731
+ ),
732
+ "math_eq": r"\log p(y\mid z) = \sum_j \left[-\frac{(\mu_j(z)-y_j)^2}{2\,(1+\sigma_j^2(z))} - \frac{1}{2}\log\left(2\pi\,(1+\sigma_j^2(z))\right)\right]",
733
+ "guardrails": (
734
+ "The likelihood includes the GP predictive variance `σ_j²(z)` (the whitened observation noise is the "
735
+ "identity), so the posterior widens where the surrogate is uncertain, and keeps the normalizing "
736
+ "log(2π·v) term because ST-MCMC needs a properly normalized density. TMCMC runs with burn-in and "
737
+ "thinning below its own 20 / 3 defaults since they multiply per-stage cost. GP fits keep "
738
+ "hyperparameters inside data-relative bounds (±2 decades on length scales and signal std; noise std "
739
+ "between 1e-3 and 1 of the signal std) to avoid the near-zero-noise fits that ST-MCMC resampling "
740
+ "can collapse onto, with `Fminbox` capped at 50 inner / 3 outer iterations."
741
+ ),
742
+ "params": [
743
+ {"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"},
744
+ {"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"},
745
+ {"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the TMCMC draws)"},
746
+ {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
747
+ {"Parameter": "retain_var (output whitening)", "Value": "0.99"},
748
+ {"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"},
749
+ ],
750
+ "notes": (
751
+ "**The batch is a plain draw from the ST-MCMC posterior, with no batch optimization.** TMCMC cost grows roughly linearly with "
752
+ "the population size, and a small population gives a degenerate posterior in the high-dimensional "
753
+ "L96 cases. In the current UQ data (c = 1.5, best ensemble size) its mean budget is 6.4 on L63 and 5.6 on L96 "
754
+ "const-force, but on L96 vec-force it is 240 and it fails on at least 85% of seeds at every ensemble size."
755
+ ),
756
+ },
757
  }
758
 
759
  # KNOWN_METHODS is the source of truth for the abbreviation/full-name list,
src/pages/UQLeaderboard.py CHANGED
@@ -35,7 +35,6 @@ render_leaderboard(
35
  state_prefix="uq",
36
  default_target=1.5,
37
  show_failure_panel=True,
38
- show_scoring_modes=False,
39
  canonical_target_levels=UQ_TARGET_LEVELS,
40
  budget_store=load_uq_budget_store(),
41
  benchmark_dims=BENCHMARK_DIMS,
 
35
  state_prefix="uq",
36
  default_target=1.5,
37
  show_failure_panel=True,
 
38
  canonical_target_levels=UQ_TARGET_LEVELS,
39
  budget_store=load_uq_budget_store(),
40
  benchmark_dims=BENCHMARK_DIMS,
src/streamlit_app.py CHANGED
@@ -5,11 +5,11 @@ import pandas as pd
5
  import streamlit as st
6
 
7
  try:
8
- from common.method_registry import KNOWN_METHODS
9
  from common.leaderboard import _UPDATE_TYPE_ORDER
10
  except ModuleNotFoundError:
11
  sys.path.append(str(Path(__file__).resolve().parent))
12
- from common.method_registry import KNOWN_METHODS
13
  from common.leaderboard import _UPDATE_TYPE_ORDER
14
 
15
  st.set_page_config(page_title="Calibration Benchmark", page_icon="🏠", layout="wide")
@@ -156,6 +156,18 @@ st.caption(
156
  "general), not whether a method is 'Bayesian'."
157
  )
158
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  st.subheader("Key metric")
160
  st.markdown(
161
  "The reported metric is the **mean number of forward-model evaluations** required "
 
5
  import streamlit as st
6
 
7
  try:
8
+ from common.method_registry import KNOWN_METHODS, SUITABILITY_FAMILIES, family_full_name
9
  from common.leaderboard import _UPDATE_TYPE_ORDER
10
  except ModuleNotFoundError:
11
  sys.path.append(str(Path(__file__).resolve().parent))
12
+ from common.method_registry import KNOWN_METHODS, SUITABILITY_FAMILIES, family_full_name
13
  from common.leaderboard import _UPDATE_TYPE_ORDER
14
 
15
  st.set_page_config(page_title="Calibration Benchmark", page_icon="🏠", layout="wide")
 
156
  "general), not whether a method is 'Bayesian'."
157
  )
158
 
159
+ _family_key_rows = [
160
+ {"Family": family_full_name(fam), "Algorithms": ", ".join(members)}
161
+ for fam, members in SUITABILITY_FAMILIES.items()
162
+ ]
163
+ st.markdown("**Method families** — algorithms of similar construction, used to group the suitability tables and method selectors:")
164
+ st.dataframe(
165
+ pd.DataFrame(_family_key_rows),
166
+ hide_index=True,
167
+ use_container_width=True,
168
+ height=int((len(_family_key_rows) + 1) * 35.2 + 3),
169
+ )
170
+
171
  st.subheader("Key metric")
172
  st.markdown(
173
  "The reported metric is the **mean number of forward-model evaluations** required "