Separate GBOED from IES and rerun
#19
by odunbar - opened
- .cache/known_methods_snapshot.json +47 -6
- .claude/skills/leaderboard-manager/references/manage-display.md +10 -25
- CLAUDE.md +27 -2
- data/{gboed-uq_results/boed_l63_ensemble_results_2026-07-30_minimal.nc → adam_results/leaderboard_adam_l96_flux-force_2026-10-21.nc} +2 -2
- data/gboed-uq_results/{boed_l96_const-force_2026-07-30_minimal.nc → boed-iekf-eig_l63_ensemble_results_2026-10-19_minimal.nc} +2 -2
- data/gboed-uq_results/{boed_l96_vec-force_2026-08-07_minimal.nc → boed-iekf-eig_l96_const-force_2026-10-19_minimal.nc} +2 -2
- data/gboed-uq_results/boed-iekf-eig_l96_flux-force_2026-10-19_minimal.nc +3 -0
- data/gboed-uq_results/boed-iekf-eig_l96_vec-force_2026-10-19_minimal.nc +3 -0
- data/gboed-uq_results/boed-iekf_l63_ensemble_results_2026-10-19_minimal.nc +3 -0
- data/gboed-uq_results/boed-iekf_l96_const-force_2026-10-19_minimal.nc +3 -0
- data/gboed-uq_results/boed-iekf_l96_flux-force_2026-10-19_minimal.nc +3 -0
- data/gboed-uq_results/boed-iekf_l96_vec-force_2026-10-19_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc-eig_l63_ensemble_results_2026-10-16_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc-eig_l96_const-force_2026-10-16_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc-eig_l96_flux-force_2026-10-16_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc-eig_l96_vec-force_2026-10-16_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc_l63_ensemble_results_2026-10-16_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc_l96_const-force_2026-10-16_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc_l96_flux-force_2026-10-16_minimal.nc +3 -0
- data/gboed-uq_results/boed-tmcmc_l96_vec-force_2026-10-16_minimal.nc +3 -0
- data/levenberg_marquardt_results/leaderboard_lm_l96_flux-force_2026-10-20.nc +3 -0
- src/common/leaderboard.py +86 -120
- src/common/method_registry.py +66 -4
- src/data_store.py +18 -3
- src/pages/MethodDetails.py +135 -3
- src/pages/UQLeaderboard.py +0 -1
- src/streamlit_app.py +14 -2
.cache/known_methods_snapshot.json
CHANGED
|
@@ -126,16 +126,20 @@
|
|
| 126 |
"gradient_descent"
|
| 127 |
]
|
| 128 |
},
|
| 129 |
-
"gboed": {
|
| 130 |
-
"abbreviation": "GBOED",
|
| 131 |
-
"Method": "Goal-oriented Bayesian Optimal Experimental Design",
|
| 132 |
"parallelism": "parallel-interacting",
|
| 133 |
"update_type": "general",
|
| 134 |
"method_goal": "uq",
|
| 135 |
"emulator_use": "within-optimize",
|
| 136 |
"aliases": [
|
| 137 |
"gboed",
|
| 138 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
]
|
| 140 |
},
|
| 141 |
"lm-kgmm": {
|
|
@@ -193,7 +197,7 @@
|
|
| 193 |
"rl-ddpg-short": {
|
| 194 |
"abbreviation": "RL-DDPG-SHORT",
|
| 195 |
"Method": "Reinforcement Learning (DDPG, short RL step)",
|
| 196 |
-
"parallelism": "serial",
|
| 197 |
"update_type": "gradient (score)",
|
| 198 |
"method_goal": "optimization",
|
| 199 |
"emulator_use": "none",
|
|
@@ -216,7 +220,7 @@
|
|
| 216 |
"rl-td3-short": {
|
| 217 |
"abbreviation": "RL-TD3-SHORT",
|
| 218 |
"Method": "Reinforcement Learning (TD3, short RL step)",
|
| 219 |
-
"parallelism": "serial",
|
| 220 |
"update_type": "gradient (score)",
|
| 221 |
"method_goal": "optimization",
|
| 222 |
"emulator_use": "none",
|
|
@@ -235,6 +239,43 @@
|
|
| 235 |
"rl-td3",
|
| 236 |
"rl-td3-long"
|
| 237 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
}
|
| 239 |
},
|
| 240 |
"observed_methods": [
|
|
|
|
| 126 |
"gradient_descent"
|
| 127 |
]
|
| 128 |
},
|
| 129 |
+
"gboed-tmcmc": {
|
| 130 |
+
"abbreviation": "GBOED-TMCMC",
|
| 131 |
+
"Method": "Goal-oriented Bayesian Optimal Experimental Design (TMCMC)",
|
| 132 |
"parallelism": "parallel-interacting",
|
| 133 |
"update_type": "general",
|
| 134 |
"method_goal": "uq",
|
| 135 |
"emulator_use": "within-optimize",
|
| 136 |
"aliases": [
|
| 137 |
"gboed",
|
| 138 |
+
"gboed-tmcmc",
|
| 139 |
+
"boed",
|
| 140 |
+
"gboed-tmcmc-eig",
|
| 141 |
+
"boed-tmcmc-eig",
|
| 142 |
+
"tmcmc-eig"
|
| 143 |
]
|
| 144 |
},
|
| 145 |
"lm-kgmm": {
|
|
|
|
| 197 |
"rl-ddpg-short": {
|
| 198 |
"abbreviation": "RL-DDPG-SHORT",
|
| 199 |
"Method": "Reinforcement Learning (DDPG, short RL step)",
|
| 200 |
+
"parallelism": "serial (short window)",
|
| 201 |
"update_type": "gradient (score)",
|
| 202 |
"method_goal": "optimization",
|
| 203 |
"emulator_use": "none",
|
|
|
|
| 220 |
"rl-td3-short": {
|
| 221 |
"abbreviation": "RL-TD3-SHORT",
|
| 222 |
"Method": "Reinforcement Learning (TD3, short RL step)",
|
| 223 |
+
"parallelism": "serial (short window)",
|
| 224 |
"update_type": "gradient (score)",
|
| 225 |
"method_goal": "optimization",
|
| 226 |
"emulator_use": "none",
|
|
|
|
| 239 |
"rl-td3",
|
| 240 |
"rl-td3-long"
|
| 241 |
]
|
| 242 |
+
},
|
| 243 |
+
"ies-iekf": {
|
| 244 |
+
"abbreviation": "IES-IEKF",
|
| 245 |
+
"Method": "Iterated Emulate-Sample (IEKF)",
|
| 246 |
+
"parallelism": "parallel-interacting",
|
| 247 |
+
"update_type": "general",
|
| 248 |
+
"method_goal": "uq",
|
| 249 |
+
"emulator_use": "within-optimize",
|
| 250 |
+
"aliases": [
|
| 251 |
+
"ies-iekf",
|
| 252 |
+
"boed-iekf"
|
| 253 |
+
]
|
| 254 |
+
},
|
| 255 |
+
"gboed-iekf": {
|
| 256 |
+
"abbreviation": "GBOED-IEKF",
|
| 257 |
+
"Method": "Goal-oriented Bayesian Optimal Experimental Design (IEKF)",
|
| 258 |
+
"parallelism": "parallel-interacting",
|
| 259 |
+
"update_type": "general",
|
| 260 |
+
"method_goal": "uq",
|
| 261 |
+
"emulator_use": "within-optimize",
|
| 262 |
+
"aliases": [
|
| 263 |
+
"gboed-iekf",
|
| 264 |
+
"gboed-iekf-eig",
|
| 265 |
+
"boed-iekf-eig"
|
| 266 |
+
]
|
| 267 |
+
},
|
| 268 |
+
"ies-tmcmc": {
|
| 269 |
+
"abbreviation": "IES-TMCMC",
|
| 270 |
+
"Method": "Iterated Emulate-Sample (TMCMC)",
|
| 271 |
+
"parallelism": "parallel-interacting",
|
| 272 |
+
"update_type": "general",
|
| 273 |
+
"method_goal": "uq",
|
| 274 |
+
"emulator_use": "within-optimize",
|
| 275 |
+
"aliases": [
|
| 276 |
+
"ies-tmcmc",
|
| 277 |
+
"boed-tmcmc"
|
| 278 |
+
]
|
| 279 |
}
|
| 280 |
},
|
| 281 |
"observed_methods": [
|
.claude/skills/leaderboard-manager/references/manage-display.md
CHANGED
|
@@ -23,9 +23,8 @@ next time someone edits the file.
|
|
| 23 |
2. Benchmark selector (`st.selectbox("Benchmark", ...)`) — always a single benchmark;
|
| 24 |
there is **no "All benchmarks" option**. Every chart and the table below operate on
|
| 25 |
whatever `selected_benchmark` is currently chosen.
|
| 26 |
-
3. **Controls — always visible**, not collapsed. Look for `st.subheader("
|
| 27 |
-
|
| 28 |
-
when "Custom Blend" is selected), and the "Methods to display in charts" pills
|
| 29 |
selector (see "Methods selector" below). These used to be wrapped in
|
| 30 |
`st.expander(..., expanded=False)`; that was removed so users don't have to open a
|
| 31 |
dropdown to see or change the target level.
|
|
@@ -67,7 +66,7 @@ list (not one-per-view):
|
|
| 67 |
table_column_order = [
|
| 68 |
"Placement", "abbreviation", "Method",
|
| 69 |
"update_type", "parallelism", "method_goal", "emulator_use", "Score",
|
| 70 |
-
"Mean Forward Model Runs",
|
| 71 |
"Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
|
| 72 |
]
|
| 73 |
```
|
|
@@ -106,28 +105,14 @@ To add it:
|
|
| 106 |
|
| 107 |
---
|
| 108 |
|
| 109 |
-
## Scoring
|
| 110 |
-
|
| 111 |
-
The four scoring modes are defined in `scoring_options` (a list literal near the top of
|
| 112 |
-
`render_leaderboard()`, right after `target_options` is computed):
|
| 113 |
-
```python
|
| 114 |
-
scoring_options = [
|
| 115 |
-
"Mean Forward Model Runs",
|
| 116 |
-
"Minimum Forward Model Runs",
|
| 117 |
-
"Smallest Optimal Ensemble Size",
|
| 118 |
-
"Custom Blend",
|
| 119 |
-
]
|
| 120 |
-
```
|
| 121 |
-
|
| 122 |
-
Each mode maps to a sort key and a score column assignment inside `build_scored_table()`
|
| 123 |
-
— search for the `if scoring_mode == "Mean Forward Model Runs": ... elif ...` chain. To
|
| 124 |
-
**add a new scoring mode**, add its name to `scoring_options` and add an `elif` branch in
|
| 125 |
-
that chain that sets `scored_df["Score"]`, `sort_columns`, and `ascending`.
|
| 126 |
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
`
|
| 130 |
-
|
|
|
|
|
|
|
| 131 |
|
| 132 |
---
|
| 133 |
|
|
|
|
| 23 |
2. Benchmark selector (`st.selectbox("Benchmark", ...)`) — always a single benchmark;
|
| 24 |
there is **no "All benchmarks" option**. Every chart and the table below operate on
|
| 25 |
whatever `selected_benchmark` is currently chosen.
|
| 26 |
+
3. **Controls — always visible**, not collapsed. Look for `st.subheader("Target &
|
| 27 |
+
Method Controls")`: target-level radio and the "Methods to display in charts" pills
|
|
|
|
| 28 |
selector (see "Methods selector" below). These used to be wrapped in
|
| 29 |
`st.expander(..., expanded=False)`; that was removed so users don't have to open a
|
| 30 |
dropdown to see or change the target level.
|
|
|
|
| 66 |
table_column_order = [
|
| 67 |
"Placement", "abbreviation", "Method",
|
| 68 |
"update_type", "parallelism", "method_goal", "emulator_use", "Score",
|
| 69 |
+
"Mean Forward Model Runs",
|
| 70 |
"Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
|
| 71 |
]
|
| 72 |
```
|
|
|
|
| 105 |
|
| 106 |
---
|
| 107 |
|
| 108 |
+
## Scoring
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
+
There is a single scoring basis: mean forward-model runs. The scoring-mode radio, the
|
| 111 |
+
"Custom Blend" slider and the minimum-runs / ensemble-size modes were removed. In
|
| 112 |
+
`build_scored_table()`, `Score` is the normalized 0-100 form of `Mean Forward Model Runs`
|
| 113 |
+
(`mean_runs_score`), and rows are sorted by reliability, then mean runs, then optimal
|
| 114 |
+
ensemble size. To reintroduce another mode, add a control above the target radio and branch
|
| 115 |
+
on it where `Score` / `sort_columns` are set.
|
| 116 |
|
| 117 |
---
|
| 118 |
|
CLAUDE.md
CHANGED
|
@@ -57,6 +57,8 @@ calibration_benchmark/
|
|
| 57 |
│ │ │ when budget_store is None (Optimization board), instead renders a
|
| 58 |
│ │ │ "Mean Iterations to Reach Target" chart derived as metric/ensemble_size
|
| 59 |
│ │ └── method_registry.py KNOWN_METHODS registry + name canonicalization
|
|
|
|
|
|
|
| 60 |
│ └── pages/
|
| 61 |
│ ├── UQLeaderboard.py 🎯 UQ Leaderboard — calls render_leaderboard(load_uq_store())
|
| 62 |
│ ├── MethodDetails.py 📘 Method configurations — per-method (selection-driven) technical
|
|
@@ -81,9 +83,19 @@ calibration_benchmark/
|
|
| 81 |
│ ├── rl-td3/ RL-TD3-SHORT / RL-TD3 (TD3 variant of the RL experiments; RL-TD3 files are `rl-td3-long` in the NetCDF) results (per-benchmark files)
|
| 82 |
│ ├── rl-ddpg/ RL-DDPG-SHORT / RL-DDPG (DDPG actor-critic; short RL step / standard-window RL step; RL-DDPG files are `rl-ddpg-long` in the NetCDF) results (per-benchmark files)
|
| 83 |
│ ├── ces-eki-dmc_results/ CES-EKI-DMC ensemble-results files (UQ source)
|
| 84 |
-
│ ├── gboed-uq_results/ GBOED ensemble-results files (UQ source)
|
| 85 |
│ ├── gnki-uq_results/ IEKF (a.k.a GNKI) ensemble-results files (UQ source)
|
| 86 |
│ └── abc/ ABC ensemble-results files (UQ source; per-benchmark files)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
├── examples/ lorenz_demo.ipynb + lorenz.py (reference, not app code)
|
| 88 |
├── From_Rob_6-10/ Raw collaborator data drop — NOT read by the app
|
| 89 |
├── .cache/ known_methods_snapshot.json (auto-generated, safe to delete)
|
|
@@ -204,7 +216,10 @@ A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
|
|
| 204 |
| `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | gradient (Kalman) | uq | after-optimize | — |
|
| 205 |
| `adam` | ADAM | serial | gradient (exact) | optimization | none | — |
|
| 206 |
| `lm` | LM | serial | gradient (exact) | optimization | none | gradient_descent |
|
| 207 |
-
| `gboed` | GBOED | parallel-interacting | general | uq | within-optimize | boed |
|
|
|
|
|
|
|
|
|
|
| 208 |
| `lm-kgmm` | LM-KGMM | serial | gradient (score) | optimization | none | — |
|
| 209 |
| `lm-dsm` | LM-DSM | serial | gradient (score) | optimization | none | — |
|
| 210 |
| `adam-kgmm` | ADAM-KGMM | serial | gradient (score) | optimization | none | — |
|
|
@@ -223,6 +238,10 @@ A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
|
|
| 223 |
2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
|
| 224 |
keyed by **abbreviation** (e.g. `"TEKI"`). ⚠️ Metadata is split across both files.
|
| 225 |
3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
|
| 227 |
### Add an optimization benchmark dataset
|
| 228 |
1. Add a `"BENCHMARK_NAME": [list_of_nc_paths]` entry to `DATASET_FILES` in `src/data_store.py`.
|
|
@@ -295,6 +314,12 @@ the `budget_store` parameter of `render_leaderboard()` in `src/common/leaderboar
|
|
| 295 |
- **Suitability table is fixed at `default_target`** — `_render_suitability_table()` always
|
| 296 |
evaluates suitability at the `default_target` passed to `render_leaderboard()`, independent
|
| 297 |
of what the user selects in the target radio control below it.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
- **Suitability criteria** — a method is green for a benchmark if there exists an ensemble size
|
| 299 |
with `failure_rate < 20%` and `mean_budget ≤ 3× global_best` at `default_target`. Gray means
|
| 300 |
no data for that benchmark; the method may be suitable on other benchmarks.
|
|
|
|
| 57 |
│ │ │ when budget_store is None (Optimization board), instead renders a
|
| 58 |
│ │ │ "Mean Iterations to Reach Target" chart derived as metric/ensemble_size
|
| 59 |
│ │ └── method_registry.py KNOWN_METHODS registry + name canonicalization
|
| 60 |
+
│ │ SUITABILITY_FAMILIES — method families (CES, Kalman, Gradient, GFDT, RL, IES,
|
| 61 |
+
│ │ GBOED, General) grouping rows in the online + LaTeX suitability tables
|
| 62 |
│ └── pages/
|
| 63 |
│ ├── UQLeaderboard.py 🎯 UQ Leaderboard — calls render_leaderboard(load_uq_store())
|
| 64 |
│ ├── MethodDetails.py 📘 Method configurations — per-method (selection-driven) technical
|
|
|
|
| 83 |
│ ├── rl-td3/ RL-TD3-SHORT / RL-TD3 (TD3 variant of the RL experiments; RL-TD3 files are `rl-td3-long` in the NetCDF) results (per-benchmark files)
|
| 84 |
│ ├── rl-ddpg/ RL-DDPG-SHORT / RL-DDPG (DDPG actor-critic; short RL step / standard-window RL step; RL-DDPG files are `rl-ddpg-long` in the NetCDF) results (per-benchmark files)
|
| 85 |
│ ├── ces-eki-dmc_results/ CES-EKI-DMC ensemble-results files (UQ source)
|
| 86 |
+
│ ├── gboed-uq_results/ GBOED ensemble-results files (UQ source; 2026-10-16 GBOED-TMCMC/IES-TMCMC and 2026-10-19 IEKF-based variants, all four benchmarks; files are named `boed-*`: `tmcmc-eig` = GBOED-TMCMC, `iekf-eig` = GBOED-IEKF, `iekf` = IES-IEKF, `tmcmc` = IES-TMCMC; IES = Iterated Emulate-Sample)
|
| 87 |
│ ├── gnki-uq_results/ IEKF (a.k.a GNKI) ensemble-results files (UQ source)
|
| 88 |
│ └── abc/ ABC ensemble-results files (UQ source; per-benchmark files)
|
| 89 |
+
├── offline-plots/ leaderboard_plot.py — matplotlib replica of the leaderboard charts
|
| 90 |
+
│ (failure rate, forward runs, iterations) for one benchmark/target,
|
| 91 |
+
│ serial vs ensemble panels, colors from METHOD_COLORS. Defaults to the
|
| 92 |
+
│ suitability-table target (1.1 opt / 1.5 UQ); writes to figures/.
|
| 93 |
+
│ `python offline-plots/leaderboard_plot.py [--board uq] [--methods ...]`
|
| 94 |
+
│ suitability_tables.py — regenerates tables/suitability_tables.tex (LaTeX
|
| 95 |
+
│ suitability grids, rows grouped by method family with \midrule) from the
|
| 96 |
+
│ current stores; `python offline-plots/suitability_tables.py`. New methods
|
| 97 |
+
│ must be listed in `SUITABILITY_FAMILIES` (method_registry.py) or it exits with an error.
|
| 98 |
+
├── tables/ suitability_tables.tex (generated), method_taxonomy_tables.tex (hand-written)
|
| 99 |
├── examples/ lorenz_demo.ipynb + lorenz.py (reference, not app code)
|
| 100 |
├── From_Rob_6-10/ Raw collaborator data drop — NOT read by the app
|
| 101 |
├── .cache/ known_methods_snapshot.json (auto-generated, safe to delete)
|
|
|
|
| 216 |
| `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | gradient (Kalman) | uq | after-optimize | — |
|
| 217 |
| `adam` | ADAM | serial | gradient (exact) | optimization | none | — |
|
| 218 |
| `lm` | LM | serial | gradient (exact) | optimization | none | gradient_descent |
|
| 219 |
+
| `gboed-tmcmc` | GBOED-TMCMC | parallel-interacting | general | uq | within-optimize | gboed, boed, tmcmc-eig, boed-tmcmc-eig, gboed-tmcmc-eig |
|
| 220 |
+
| `ies-iekf` | IES-IEKF | parallel-interacting | general | uq | within-optimize | boed-iekf |
|
| 221 |
+
| `gboed-iekf` | GBOED-IEKF | parallel-interacting | general | uq | within-optimize | gboed-iekf-eig, boed-iekf-eig |
|
| 222 |
+
| `ies-tmcmc` | IES-TMCMC | parallel-interacting | general | uq | within-optimize | boed-tmcmc |
|
| 223 |
| `lm-kgmm` | LM-KGMM | serial | gradient (score) | optimization | none | — |
|
| 224 |
| `lm-dsm` | LM-DSM | serial | gradient (score) | optimization | none | — |
|
| 225 |
| `adam-kgmm` | ADAM-KGMM | serial | gradient (score) | optimization | none | — |
|
|
|
|
| 238 |
2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
|
| 239 |
keyed by **abbreviation** (e.g. `"TEKI"`). ⚠️ Metadata is split across both files.
|
| 240 |
3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
|
| 241 |
+
4. Add the abbreviation to a family in `SUITABILITY_FAMILIES` (`method_registry.py`; spelled-out family
|
| 242 |
+
names live in `FAMILY_FULL_NAMES` — add an entry there for any new family). Unlisted methods
|
| 243 |
+
appear in a trailing "Other" block of the online suitability table, and
|
| 244 |
+
`offline-plots/suitability_tables.py` refuses to run until they are assigned.
|
| 245 |
|
| 246 |
### Add an optimization benchmark dataset
|
| 247 |
1. Add a `"BENCHMARK_NAME": [list_of_nc_paths]` entry to `DATASET_FILES` in `src/data_store.py`.
|
|
|
|
| 314 |
- **Suitability table is fixed at `default_target`** — `_render_suitability_table()` always
|
| 315 |
evaluates suitability at the `default_target` passed to `render_leaderboard()`, independent
|
| 316 |
of what the user selects in the target radio control below it.
|
| 317 |
+
- **Family display names** — `SUITABILITY_FAMILIES` keys are short ids (`RL`, `GFDT`, ...); the UI shows `family_full_name(key)` from `FAMILY_FULL_NAMES` (suitability table, the family key under the home-page method table, and the per-family rows of the "Methods to display in charts" pills on both leaderboards; session keys `<prefix>_selected_methods__<family>`, with `<prefix>_selected_methods` mirroring the union).
|
| 318 |
+
- **Suitability table families** — rows of the suitability table are grouped by `SUITABILITY_FAMILIES`
|
| 319 |
+
(family order = dict order, row order = list order). Online (`_render_suitability_table`) this shows as a
|
| 320 |
+
leading "Family" column labelled on the first row of each block with alternating gray shading, since
|
| 321 |
+
`st.dataframe` cannot draw rules; the LaTeX tables use `\midrule` between blocks. A family only appears on a
|
| 322 |
+
board where one of its methods has data.
|
| 323 |
- **Suitability criteria** — a method is green for a benchmark if there exists an ensemble size
|
| 324 |
with `failure_rate < 20%` and `mean_budget ≤ 3× global_best` at `default_target`. Gray means
|
| 325 |
no data for that benchmark; the method may be suitable on other benchmarks.
|
data/{gboed-uq_results/boed_l63_ensemble_results_2026-07-30_minimal.nc → adam_results/leaderboard_adam_l96_flux-force_2026-10-21.nc}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:539eb03a55d4ffe0a51b3b45db3b5dc0d0e159c8132e71639ba2eb5c0f72af95
|
| 3 |
+
size 11440
|
data/gboed-uq_results/{boed_l96_const-force_2026-07-30_minimal.nc → boed-iekf-eig_l63_ensemble_results_2026-10-19_minimal.nc}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f65ab0ae11bc8774dff077565a96dea07329cc1fe9c1055e01de4942bbb609f2
|
| 3 |
+
size 728387
|
data/gboed-uq_results/{boed_l96_vec-force_2026-08-07_minimal.nc → boed-iekf-eig_l96_const-force_2026-10-19_minimal.nc}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:04951c1aa8553181100c69f4819c3df54d72a39c4d614624a56d778cef85c710
|
| 3 |
+
size 728387
|
data/gboed-uq_results/boed-iekf-eig_l96_flux-force_2026-10-19_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4d65fc5289eece6735841ad4024b2fa3c9ec3d09e4a6c0442e3ef217399a9941
|
| 3 |
+
size 728387
|
data/gboed-uq_results/boed-iekf-eig_l96_vec-force_2026-10-19_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:af1d2f624baf2e6c398718fa593caabccf7db232a133041bd3ed46eef28266a5
|
| 3 |
+
size 728387
|
data/gboed-uq_results/boed-iekf_l63_ensemble_results_2026-10-19_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a39050677da137a8cb4ab906c5e4d6f532c0a4ef9c7c7dd0c60885e988554d48
|
| 3 |
+
size 728383
|
data/gboed-uq_results/boed-iekf_l96_const-force_2026-10-19_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d2fdf0355c4c2461ff96c1bb610a472fa9c4f5379b55a748f74eb089daf5e2be
|
| 3 |
+
size 728383
|
data/gboed-uq_results/boed-iekf_l96_flux-force_2026-10-19_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a61dab8820a1d681da06758a13ade609b2feeb1d0b9b95b645d858dd504d368d
|
| 3 |
+
size 728383
|
data/gboed-uq_results/boed-iekf_l96_vec-force_2026-10-19_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f2779b4eec62487f8794e472fcf7abc3d2721549ce7d857c82f9ced4075551d7
|
| 3 |
+
size 728383
|
data/gboed-uq_results/boed-tmcmc-eig_l63_ensemble_results_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:add03f4917aeff93b36b051520b1341363c494905df5352c2ecdafa46d8be19a
|
| 3 |
+
size 728388
|
data/gboed-uq_results/boed-tmcmc-eig_l96_const-force_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c2312bf2553c4fb38c47f4ab50eb0f2d339d9af21dbcdb08ec549d224eaeee79
|
| 3 |
+
size 728388
|
data/gboed-uq_results/boed-tmcmc-eig_l96_flux-force_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0ea5592248a9ef9f739d95825cdfe9a01fd76c6c6c50ecf2e341139e26af4d4a
|
| 3 |
+
size 728388
|
data/gboed-uq_results/boed-tmcmc-eig_l96_vec-force_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e5c7924216ef9add1bd56e294d7bded10c7b0ca2df696b9c3f2ceec66bc2e8a9
|
| 3 |
+
size 728388
|
data/gboed-uq_results/boed-tmcmc_l63_ensemble_results_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7ed95ad74202e40bcf6b412f37d2f35b4d8b5a66f1ae9effee9dac19f7068aaa
|
| 3 |
+
size 728384
|
data/gboed-uq_results/boed-tmcmc_l96_const-force_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f5072b6ec954baeb5add8097a29a33ca82e4d55a9f81965d43ee2bb629e5e644
|
| 3 |
+
size 728384
|
data/gboed-uq_results/boed-tmcmc_l96_flux-force_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5cd0f8903089ebdb468e5b3e41ce08c89207905ef1f47135f1e5369fa1632203
|
| 3 |
+
size 728384
|
data/gboed-uq_results/boed-tmcmc_l96_vec-force_2026-10-16_minimal.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9ac1182e23750b0db31239f52c8279771c8d6cbcfe66743184d2ffaa15b30194
|
| 3 |
+
size 728384
|
data/levenberg_marquardt_results/leaderboard_lm_l96_flux-force_2026-10-20.nc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e2a216e92286052411116fc527cb1d49e66272e522f48aeaf1d985aa3495976
|
| 3 |
+
size 11440
|
src/common/leaderboard.py
CHANGED
|
@@ -25,13 +25,16 @@ import pandas as pd
|
|
| 25 |
import streamlit as st
|
| 26 |
|
| 27 |
try:
|
| 28 |
-
from common.method_registry import METHOD_COLORS
|
| 29 |
except ModuleNotFoundError:
|
| 30 |
-
from src.common.method_registry import METHOD_COLORS
|
| 31 |
|
| 32 |
_SUITABLE = "#009E73" # Okabe-Ito teal-green (colorblind-safe)
|
| 33 |
_UNSUITABLE = "#C0392B" # dark red
|
| 34 |
_UNTESTED = "#BDBDBD" # gray
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
|
| 37 |
def _method_color(present_abbrevs: list[str]) -> alt.Color:
|
|
@@ -67,6 +70,24 @@ _UPDATE_TYPE_ORDER: dict[str, int] = {
|
|
| 67 |
_RELIABLE_FAILURE_THRESHOLD = 20.0
|
| 68 |
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
def _render_suitability_table(
|
| 71 |
store: pd.DataFrame,
|
| 72 |
target_col: str,
|
|
@@ -80,16 +101,20 @@ def _render_suitability_table(
|
|
| 80 |
store["benchmark"].unique().tolist(),
|
| 81 |
key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
|
| 82 |
)
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
)
|
| 89 |
methods = sorted(
|
| 90 |
store["abbreviation"].dropna().unique().tolist(),
|
| 91 |
-
key=lambda a: (
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
)
|
|
|
|
| 93 |
|
| 94 |
target_str = str(float(suitability_target))
|
| 95 |
target_str_col = f"{target_col}_str"
|
|
@@ -144,13 +169,27 @@ def _render_suitability_table(
|
|
| 144 |
display_df = pd.DataFrame(cell_text).T.rename(columns=col_labels)
|
| 145 |
color_df = pd.DataFrame(cell_color).T.rename(columns=col_labels)
|
| 146 |
display_df.index.name = "Method"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
|
| 148 |
def _style(df: pd.DataFrame) -> pd.DataFrame:
|
| 149 |
result = pd.DataFrame("", index=df.index, columns=df.columns)
|
| 150 |
for row in df.index:
|
| 151 |
for col in df.columns:
|
| 152 |
bg = color_df.loc[row, col]
|
| 153 |
-
fg = "#212529" if bg == _UNTESTED else "white"
|
| 154 |
result.loc[row, col] = (
|
| 155 |
f"background-color: {bg}; color: {fg}; "
|
| 156 |
"text-align: center; font-weight: bold"
|
|
@@ -165,7 +204,8 @@ def _render_suitability_table(
|
|
| 165 |
"**Red** (failed): data present but no ensemble size met the criteria; "
|
| 166 |
"the percentage is the lowest failure rate (share of runs that did not reach the target) "
|
| 167 |
"across ensemble sizes. "
|
| 168 |
-
"**Gray (—)**: no data for this benchmark."
|
|
|
|
| 169 |
)
|
| 170 |
st.dataframe(
|
| 171 |
display_df.style.apply(_style, axis=None),
|
|
@@ -184,7 +224,6 @@ def render_leaderboard(
|
|
| 184 |
default_target: float,
|
| 185 |
raw_page: str | None = None,
|
| 186 |
show_failure_panel: bool = False,
|
| 187 |
-
show_scoring_modes: bool = True,
|
| 188 |
canonical_target_levels: list[float] | None = None,
|
| 189 |
budget_store: pd.DataFrame | None = None,
|
| 190 |
benchmark_dims: dict[str, tuple[int, int, int]] | None = None,
|
|
@@ -220,8 +259,6 @@ def render_leaderboard(
|
|
| 220 |
show_failure_panel:
|
| 221 |
If ``True``, render a grouped-bar failure-rate chart below the main
|
| 222 |
performance chart.
|
| 223 |
-
show_scoring_modes:
|
| 224 |
-
If ``True``, show scoring-mode radio controls.
|
| 225 |
canonical_target_levels:
|
| 226 |
If provided, the target-level selector always offers exactly these
|
| 227 |
values (as strings) regardless of what is present in the data. Use
|
|
@@ -263,18 +300,9 @@ def render_leaderboard(
|
|
| 263 |
metric_store[target_col].astype(str).unique().tolist()
|
| 264 |
)
|
| 265 |
|
| 266 |
-
scoring_options = [
|
| 267 |
-
"Mean Forward Model Runs",
|
| 268 |
-
"Minimum Forward Model Runs",
|
| 269 |
-
"Smallest Optimal Ensemble Size",
|
| 270 |
-
"Custom Blend",
|
| 271 |
-
]
|
| 272 |
-
|
| 273 |
# Session-state keys namespaced by state_prefix so two leaderboard pages
|
| 274 |
# don't share control state.
|
| 275 |
k_target = f"{state_prefix}_selected_target"
|
| 276 |
-
k_scoring = f"{state_prefix}_scoring_mode"
|
| 277 |
-
k_weight = f"{state_prefix}_fwdruns_weight_percent"
|
| 278 |
k_methods = f"{state_prefix}_selected_methods"
|
| 279 |
|
| 280 |
default_target_str = str(float(default_target))
|
|
@@ -286,17 +314,6 @@ def render_leaderboard(
|
|
| 286 |
)
|
| 287 |
selected_target = st.session_state[k_target]
|
| 288 |
|
| 289 |
-
current_scoring_mode = st.session_state.get(k_scoring, "Mean Forward Model Runs")
|
| 290 |
-
if current_scoring_mode not in scoring_options:
|
| 291 |
-
current_scoring_mode = scoring_options[0]
|
| 292 |
-
|
| 293 |
-
current_fwdruns_weight_percent = int(st.session_state.get(k_weight, 80))
|
| 294 |
-
current_fwdruns_weight_percent = max(0, min(100, current_fwdruns_weight_percent))
|
| 295 |
-
|
| 296 |
-
scoring_mode = current_scoring_mode
|
| 297 |
-
fwdruns_weight = current_fwdruns_weight_percent / 100.0
|
| 298 |
-
ensemble_weight = 1.0 - fwdruns_weight
|
| 299 |
-
|
| 300 |
def build_scored_table(input_df: pd.DataFrame, add_rank: bool = True) -> pd.DataFrame:
|
| 301 |
ranking_source = input_df[input_df[target_str_col] == selected_target]
|
| 302 |
if ranking_source.empty:
|
|
@@ -316,7 +333,6 @@ def render_leaderboard(
|
|
| 316 |
["algorithm_type", "abbreviation", "Method"], as_index=False
|
| 317 |
).agg(
|
| 318 |
**{"Mean Forward Model Runs": ("metric", "mean")},
|
| 319 |
-
**{"Minimum Forward Model Runs": ("metric", "min")},
|
| 320 |
**{"Ensemble Sizes Used": ("ensemble_size", "nunique")},
|
| 321 |
)
|
| 322 |
|
|
@@ -351,7 +367,6 @@ def render_leaderboard(
|
|
| 351 |
|
| 352 |
scored_df["Optimal Ensemble Size"] = scored_df["Optimal Ensemble Size"].round(2)
|
| 353 |
scored_df["Mean Forward Model Runs"] = scored_df["Mean Forward Model Runs"].round(4)
|
| 354 |
-
scored_df["Minimum Forward Model Runs"] = scored_df["Minimum Forward Model Runs"].round(4)
|
| 355 |
scored_df["Mean Failure Rate (%)"] = scored_df["Mean Failure Rate (%)"].round(1)
|
| 356 |
|
| 357 |
mean_runs_min = scored_df["Mean Forward Model Runs"].min()
|
|
@@ -363,47 +378,13 @@ def render_leaderboard(
|
|
| 363 |
else:
|
| 364 |
scored_df["mean_runs_score"] = 100.0
|
| 365 |
|
| 366 |
-
minimum_runs_min = scored_df["Minimum Forward Model Runs"].min()
|
| 367 |
-
minimum_runs_max = scored_df["Minimum Forward Model Runs"].max()
|
| 368 |
-
if minimum_runs_max > minimum_runs_min:
|
| 369 |
-
scored_df["minimum_runs_score"] = (
|
| 370 |
-
100.0 * (minimum_runs_max - scored_df["Minimum Forward Model Runs"]) / (minimum_runs_max - minimum_runs_min)
|
| 371 |
-
)
|
| 372 |
-
else:
|
| 373 |
-
scored_df["minimum_runs_score"] = 100.0
|
| 374 |
-
|
| 375 |
-
ens_min = scored_df["Optimal Ensemble Size"].min()
|
| 376 |
-
ens_max = scored_df["Optimal Ensemble Size"].max()
|
| 377 |
-
if ens_max > ens_min:
|
| 378 |
-
scored_df["ensemble_score"] = (
|
| 379 |
-
100.0 * (ens_max - scored_df["Optimal Ensemble Size"]) / (ens_max - ens_min)
|
| 380 |
-
)
|
| 381 |
-
else:
|
| 382 |
-
scored_df["ensemble_score"] = 100.0
|
| 383 |
-
|
| 384 |
scored_df["_unreliable"] = (
|
| 385 |
scored_df["Mean Failure Rate (%)"].fillna(0.0) >= _RELIABLE_FAILURE_THRESHOLD
|
| 386 |
)
|
| 387 |
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
ascending = [True, True, True, True]
|
| 392 |
-
elif scoring_mode == "Minimum Forward Model Runs":
|
| 393 |
-
scored_df["Score"] = scored_df["minimum_runs_score"]
|
| 394 |
-
sort_columns = ["_unreliable", "Minimum Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
|
| 395 |
-
ascending = [True, True, True, True]
|
| 396 |
-
elif scoring_mode == "Smallest Optimal Ensemble Size":
|
| 397 |
-
scored_df["Score"] = scored_df["ensemble_score"]
|
| 398 |
-
sort_columns = ["_unreliable", "Optimal Ensemble Size", "Mean Forward Model Runs", "abbreviation"]
|
| 399 |
-
ascending = [True, True, True, True]
|
| 400 |
-
else:
|
| 401 |
-
scored_df["Score"] = (
|
| 402 |
-
fwdruns_weight * scored_df["mean_runs_score"]
|
| 403 |
-
+ ensemble_weight * scored_df["ensemble_score"]
|
| 404 |
-
)
|
| 405 |
-
sort_columns = ["_unreliable", "Score", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
|
| 406 |
-
ascending = [True, False, True, True, True]
|
| 407 |
|
| 408 |
scored_df = (
|
| 409 |
scored_df.sort_values(sort_columns, ascending=ascending)
|
|
@@ -426,8 +407,8 @@ def render_leaderboard(
|
|
| 426 |
if dnf_abbrevs:
|
| 427 |
dnf_rows = failure_agg[failure_agg["abbreviation"].isin(dnf_abbrevs)].copy()
|
| 428 |
for col in [
|
| 429 |
-
"Score", "mean_runs_score",
|
| 430 |
-
"Mean Forward Model Runs",
|
| 431 |
"Optimal Ensemble Size", "Ensemble Sizes Used",
|
| 432 |
]:
|
| 433 |
dnf_rows[col] = float("nan")
|
|
@@ -466,9 +447,22 @@ def render_leaderboard(
|
|
| 466 |
# All runs failed at this target (or no data) — show every method so the
|
| 467 |
# all-DNF warning/chart-suppression path still has something to work with.
|
| 468 |
default_methods = available_methods
|
| 469 |
-
|
| 470 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 471 |
st.session_state[k_methods] = valid_saved if valid_saved else default_methods
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
|
| 473 |
# Attach the four taxonomy tags (constant per abbreviation, so this is a
|
| 474 |
# plain lookup merge rather than a groupby key threaded through every
|
|
@@ -490,22 +484,12 @@ def render_leaderboard(
|
|
| 490 |
na_position="last",
|
| 491 |
).drop(columns=["_update_type_sort", "_is_dnf"]).reset_index(drop=True)
|
| 492 |
|
| 493 |
-
|
| 494 |
-
score_basis = "mean forward-model runs at the selected target level (lower is better)"
|
| 495 |
-
elif scoring_mode == "Minimum Forward Model Runs":
|
| 496 |
-
score_basis = "minimum forward-model runs at the selected target level (lower is better)"
|
| 497 |
-
elif scoring_mode == "Smallest Optimal Ensemble Size":
|
| 498 |
-
score_basis = "mean optimal ensemble size at the selected target level (lower is better)"
|
| 499 |
-
else:
|
| 500 |
-
score_basis = (
|
| 501 |
-
f"weighted blend of normalized forward-model-runs score ({fwdruns_weight:.0%}) "
|
| 502 |
-
f"and normalized ensemble-size score ({ensemble_weight:.0%})"
|
| 503 |
-
)
|
| 504 |
|
| 505 |
# Controls — always visible (not collapsed behind a dropdown) so users can
|
| 506 |
-
# change target/
|
| 507 |
# failures.
|
| 508 |
-
st.subheader("
|
| 509 |
st.radio(
|
| 510 |
target_label,
|
| 511 |
options=target_options,
|
|
@@ -513,36 +497,20 @@ def render_leaderboard(
|
|
| 513 |
key=k_target,
|
| 514 |
)
|
| 515 |
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
|
|
|
|
|
|
| 522 |
)
|
| 523 |
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
max_value=100,
|
| 529 |
-
step=5,
|
| 530 |
-
key=k_weight,
|
| 531 |
-
help=(
|
| 532 |
-
"Higher forward-runs weight prioritizes fewer model evaluations; "
|
| 533 |
-
"higher ensemble-size weight prioritizes smaller ensembles."
|
| 534 |
-
),
|
| 535 |
-
)
|
| 536 |
-
|
| 537 |
-
st.pills(
|
| 538 |
-
"Methods to display in charts",
|
| 539 |
-
options=available_methods,
|
| 540 |
-
selection_mode="multi",
|
| 541 |
-
key=k_methods,
|
| 542 |
-
help="Defaults to the top 5 methods by current rank — click to add or remove.",
|
| 543 |
-
)
|
| 544 |
-
|
| 545 |
-
selected_methods = st.session_state.get(k_methods, available_methods)
|
| 546 |
if not selected_methods:
|
| 547 |
selected_methods = available_methods
|
| 548 |
|
|
@@ -840,7 +808,6 @@ def render_leaderboard(
|
|
| 840 |
"emulator_use",
|
| 841 |
"Score",
|
| 842 |
"Mean Forward Model Runs",
|
| 843 |
-
"Minimum Forward Model Runs",
|
| 844 |
"Mean Failure Rate (%)",
|
| 845 |
"Optimal Ensemble Size",
|
| 846 |
"Ensemble Sizes Used",
|
|
@@ -868,7 +835,6 @@ def render_leaderboard(
|
|
| 868 |
"Method": st.column_config.TextColumn("Method"),
|
| 869 |
"abbreviation": st.column_config.TextColumn("Abbrev."),
|
| 870 |
"Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
|
| 871 |
-
"Minimum Forward Model Runs": st.column_config.NumberColumn("Minimum Forward Model Runs", format="%.4f"),
|
| 872 |
"Score": st.column_config.ProgressColumn("Score (0-100)", min_value=0.0, max_value=100.0, format="%.1f"),
|
| 873 |
"Optimal Ensemble Size": st.column_config.NumberColumn("Mean Optimal Ensemble Size", format="%.2f"),
|
| 874 |
"Mean Failure Rate (%)": st.column_config.NumberColumn("Mean Failure Rate (%)", format="%.1f"),
|
|
|
|
| 25 |
import streamlit as st
|
| 26 |
|
| 27 |
try:
|
| 28 |
+
from common.method_registry import METHOD_COLORS, SUITABILITY_FAMILIES, family_full_name
|
| 29 |
except ModuleNotFoundError:
|
| 30 |
+
from src.common.method_registry import METHOD_COLORS, SUITABILITY_FAMILIES, family_full_name
|
| 31 |
|
| 32 |
_SUITABLE = "#009E73" # Okabe-Ito teal-green (colorblind-safe)
|
| 33 |
_UNSUITABLE = "#C0392B" # dark red
|
| 34 |
_UNTESTED = "#BDBDBD" # gray
|
| 35 |
+
# Alternating light grays for the family-label column of the suitability table, so
|
| 36 |
+
# consecutive family blocks are visually distinct.
|
| 37 |
+
_FAMILY_BGS = ("#E9ECEF", "#CED4DA")
|
| 38 |
|
| 39 |
|
| 40 |
def _method_color(present_abbrevs: list[str]) -> alt.Color:
|
|
|
|
| 70 |
_RELIABLE_FAILURE_THRESHOLD = 20.0
|
| 71 |
|
| 72 |
|
| 73 |
+
def _group_by_family(methods: list[str]) -> list[tuple[str, list[str]]]:
|
| 74 |
+
"""Group method abbreviations into (family key, methods) in SUITABILITY_FAMILIES order.
|
| 75 |
+
|
| 76 |
+
Methods not listed in any family go into a trailing "Other" group.
|
| 77 |
+
"""
|
| 78 |
+
groups: list[tuple[str, list[str]]] = []
|
| 79 |
+
placed: set[str] = set()
|
| 80 |
+
for fam, members in SUITABILITY_FAMILIES.items():
|
| 81 |
+
present = [m for m in members if m in methods]
|
| 82 |
+
if present:
|
| 83 |
+
groups.append((fam, present))
|
| 84 |
+
placed.update(present)
|
| 85 |
+
rest = [m for m in methods if m not in placed]
|
| 86 |
+
if rest:
|
| 87 |
+
groups.append(("Other", sorted(rest)))
|
| 88 |
+
return groups
|
| 89 |
+
|
| 90 |
+
|
| 91 |
def _render_suitability_table(
|
| 92 |
store: pd.DataFrame,
|
| 93 |
target_col: str,
|
|
|
|
| 101 |
store["benchmark"].unique().tolist(),
|
| 102 |
key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
|
| 103 |
)
|
| 104 |
+
# Rows are grouped into method families (SUITABILITY_FAMILIES); methods not listed
|
| 105 |
+
# there fall into a trailing "Other" family.
|
| 106 |
+
_family_of = {m: fam for fam, ms in SUITABILITY_FAMILIES.items() for m in ms}
|
| 107 |
+
_family_rank = {fam: i for i, fam in enumerate(SUITABILITY_FAMILIES)}
|
| 108 |
+
_row_rank = {m: i for ms in SUITABILITY_FAMILIES.values() for i, m in enumerate(ms)}
|
|
|
|
| 109 |
methods = sorted(
|
| 110 |
store["abbreviation"].dropna().unique().tolist(),
|
| 111 |
+
key=lambda a: (
|
| 112 |
+
_family_rank.get(_family_of.get(a, ""), len(_family_rank)),
|
| 113 |
+
_row_rank.get(a, 0),
|
| 114 |
+
a,
|
| 115 |
+
),
|
| 116 |
)
|
| 117 |
+
family_label = {m: family_full_name(_family_of.get(m, "Other")) for m in methods}
|
| 118 |
|
| 119 |
target_str = str(float(suitability_target))
|
| 120 |
target_str_col = f"{target_col}_str"
|
|
|
|
| 169 |
display_df = pd.DataFrame(cell_text).T.rename(columns=col_labels)
|
| 170 |
color_df = pd.DataFrame(cell_color).T.rename(columns=col_labels)
|
| 171 |
display_df.index.name = "Method"
|
| 172 |
+
# Leading "Family" column, labelled only on the first row of each family block so
|
| 173 |
+
# the families read as blocks (st.dataframe cannot draw rules between row groups).
|
| 174 |
+
shown_family = [
|
| 175 |
+
fam if i == 0 or fam != family_label[methods[i - 1]] else ""
|
| 176 |
+
for i, fam in enumerate(family_label[m] for m in methods)
|
| 177 |
+
]
|
| 178 |
+
display_df.insert(0, "Family", shown_family)
|
| 179 |
+
block_idx, prev = -1, None
|
| 180 |
+
family_bg = []
|
| 181 |
+
for m in methods:
|
| 182 |
+
if family_label[m] != prev:
|
| 183 |
+
block_idx, prev = block_idx + 1, family_label[m]
|
| 184 |
+
family_bg.append(_FAMILY_BGS[block_idx % 2])
|
| 185 |
+
color_df.insert(0, "Family", family_bg)
|
| 186 |
|
| 187 |
def _style(df: pd.DataFrame) -> pd.DataFrame:
|
| 188 |
result = pd.DataFrame("", index=df.index, columns=df.columns)
|
| 189 |
for row in df.index:
|
| 190 |
for col in df.columns:
|
| 191 |
bg = color_df.loc[row, col]
|
| 192 |
+
fg = "#212529" if bg == _UNTESTED or bg in _FAMILY_BGS else "white"
|
| 193 |
result.loc[row, col] = (
|
| 194 |
f"background-color: {bg}; color: {fg}; "
|
| 195 |
"text-align: center; font-weight: bold"
|
|
|
|
| 204 |
"**Red** (failed): data present but no ensemble size met the criteria; "
|
| 205 |
"the percentage is the lowest failure rate (share of runs that did not reach the target) "
|
| 206 |
"across ensemble sizes. "
|
| 207 |
+
"**Gray (—)**: no data for this benchmark. "
|
| 208 |
+
"Rows are grouped into method families of similar construction (left column)."
|
| 209 |
)
|
| 210 |
st.dataframe(
|
| 211 |
display_df.style.apply(_style, axis=None),
|
|
|
|
| 224 |
default_target: float,
|
| 225 |
raw_page: str | None = None,
|
| 226 |
show_failure_panel: bool = False,
|
|
|
|
| 227 |
canonical_target_levels: list[float] | None = None,
|
| 228 |
budget_store: pd.DataFrame | None = None,
|
| 229 |
benchmark_dims: dict[str, tuple[int, int, int]] | None = None,
|
|
|
|
| 259 |
show_failure_panel:
|
| 260 |
If ``True``, render a grouped-bar failure-rate chart below the main
|
| 261 |
performance chart.
|
|
|
|
|
|
|
| 262 |
canonical_target_levels:
|
| 263 |
If provided, the target-level selector always offers exactly these
|
| 264 |
values (as strings) regardless of what is present in the data. Use
|
|
|
|
| 300 |
metric_store[target_col].astype(str).unique().tolist()
|
| 301 |
)
|
| 302 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
# Session-state keys namespaced by state_prefix so two leaderboard pages
|
| 304 |
# don't share control state.
|
| 305 |
k_target = f"{state_prefix}_selected_target"
|
|
|
|
|
|
|
| 306 |
k_methods = f"{state_prefix}_selected_methods"
|
| 307 |
|
| 308 |
default_target_str = str(float(default_target))
|
|
|
|
| 314 |
)
|
| 315 |
selected_target = st.session_state[k_target]
|
| 316 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
def build_scored_table(input_df: pd.DataFrame, add_rank: bool = True) -> pd.DataFrame:
|
| 318 |
ranking_source = input_df[input_df[target_str_col] == selected_target]
|
| 319 |
if ranking_source.empty:
|
|
|
|
| 333 |
["algorithm_type", "abbreviation", "Method"], as_index=False
|
| 334 |
).agg(
|
| 335 |
**{"Mean Forward Model Runs": ("metric", "mean")},
|
|
|
|
| 336 |
**{"Ensemble Sizes Used": ("ensemble_size", "nunique")},
|
| 337 |
)
|
| 338 |
|
|
|
|
| 367 |
|
| 368 |
scored_df["Optimal Ensemble Size"] = scored_df["Optimal Ensemble Size"].round(2)
|
| 369 |
scored_df["Mean Forward Model Runs"] = scored_df["Mean Forward Model Runs"].round(4)
|
|
|
|
| 370 |
scored_df["Mean Failure Rate (%)"] = scored_df["Mean Failure Rate (%)"].round(1)
|
| 371 |
|
| 372 |
mean_runs_min = scored_df["Mean Forward Model Runs"].min()
|
|
|
|
| 378 |
else:
|
| 379 |
scored_df["mean_runs_score"] = 100.0
|
| 380 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 381 |
scored_df["_unreliable"] = (
|
| 382 |
scored_df["Mean Failure Rate (%)"].fillna(0.0) >= _RELIABLE_FAILURE_THRESHOLD
|
| 383 |
)
|
| 384 |
|
| 385 |
+
scored_df["Score"] = scored_df["mean_runs_score"]
|
| 386 |
+
sort_columns = ["_unreliable", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
|
| 387 |
+
ascending = [True, True, True, True]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
|
| 389 |
scored_df = (
|
| 390 |
scored_df.sort_values(sort_columns, ascending=ascending)
|
|
|
|
| 407 |
if dnf_abbrevs:
|
| 408 |
dnf_rows = failure_agg[failure_agg["abbreviation"].isin(dnf_abbrevs)].copy()
|
| 409 |
for col in [
|
| 410 |
+
"Score", "mean_runs_score",
|
| 411 |
+
"Mean Forward Model Runs",
|
| 412 |
"Optimal Ensemble Size", "Ensemble Sizes Used",
|
| 413 |
]:
|
| 414 |
dnf_rows[col] = float("nan")
|
|
|
|
| 447 |
# All runs failed at this target (or no data) — show every method so the
|
| 448 |
# all-DNF warning/chart-suppression path still has something to work with.
|
| 449 |
default_methods = available_methods
|
| 450 |
+
# The selector is one pills widget per method family (one row each); the
|
| 451 |
+
# selection is the union of the family widgets, mirrored into k_methods.
|
| 452 |
+
method_groups = _group_by_family(available_methods)
|
| 453 |
+
family_keys = {fam: f"{k_methods}__{fam}" for fam, _ in method_groups}
|
| 454 |
+
if any(wkey in st.session_state for wkey in family_keys.values()):
|
| 455 |
+
saved_methods = [
|
| 456 |
+
m for wkey in family_keys.values() for m in st.session_state.get(wkey) or []
|
| 457 |
+
]
|
| 458 |
+
else:
|
| 459 |
+
saved_methods = default_methods
|
| 460 |
+
valid_saved = [m for m in saved_methods if m in available_methods]
|
| 461 |
st.session_state[k_methods] = valid_saved if valid_saved else default_methods
|
| 462 |
+
for fam, members in method_groups:
|
| 463 |
+
st.session_state[family_keys[fam]] = [
|
| 464 |
+
m for m in st.session_state[k_methods] if m in members
|
| 465 |
+
]
|
| 466 |
|
| 467 |
# Attach the four taxonomy tags (constant per abbreviation, so this is a
|
| 468 |
# plain lookup merge rather than a groupby key threaded through every
|
|
|
|
| 484 |
na_position="last",
|
| 485 |
).drop(columns=["_update_type_sort", "_is_dnf"]).reset_index(drop=True)
|
| 486 |
|
| 487 |
+
score_basis = "mean forward-model runs at the selected target level (lower is better)"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 488 |
|
| 489 |
# Controls — always visible (not collapsed behind a dropdown) so users can
|
| 490 |
+
# change target/methods even when the current selection yields all
|
| 491 |
# failures.
|
| 492 |
+
st.subheader("Target & Method Controls")
|
| 493 |
st.radio(
|
| 494 |
target_label,
|
| 495 |
options=target_options,
|
|
|
|
| 497 |
key=k_target,
|
| 498 |
)
|
| 499 |
|
| 500 |
+
st.markdown("**Methods to display in charts**")
|
| 501 |
+
st.caption("Defaults to the top 5 methods by current rank — click to add or remove. Rows are method families.")
|
| 502 |
+
for fam, members in method_groups:
|
| 503 |
+
st.pills(
|
| 504 |
+
family_full_name(fam),
|
| 505 |
+
options=members,
|
| 506 |
+
selection_mode="multi",
|
| 507 |
+
key=family_keys[fam],
|
| 508 |
)
|
| 509 |
|
| 510 |
+
selected_methods = [
|
| 511 |
+
m for fam, _ in method_groups for m in st.session_state.get(family_keys[fam]) or []
|
| 512 |
+
]
|
| 513 |
+
st.session_state[k_methods] = selected_methods
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 514 |
if not selected_methods:
|
| 515 |
selected_methods = available_methods
|
| 516 |
|
|
|
|
| 808 |
"emulator_use",
|
| 809 |
"Score",
|
| 810 |
"Mean Forward Model Runs",
|
|
|
|
| 811 |
"Mean Failure Rate (%)",
|
| 812 |
"Optimal Ensemble Size",
|
| 813 |
"Ensemble Sizes Used",
|
|
|
|
| 835 |
"Method": st.column_config.TextColumn("Method"),
|
| 836 |
"abbreviation": st.column_config.TextColumn("Abbrev."),
|
| 837 |
"Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
|
|
|
|
| 838 |
"Score": st.column_config.ProgressColumn("Score (0-100)", min_value=0.0, max_value=100.0, format="%.1f"),
|
| 839 |
"Optimal Ensemble Size": st.column_config.NumberColumn("Mean Optimal Ensemble Size", format="%.2f"),
|
| 840 |
"Mean Failure Rate (%)": st.column_config.NumberColumn("Mean Failure Rate (%)", format="%.1f"),
|
src/common/method_registry.py
CHANGED
|
@@ -140,14 +140,14 @@ KNOWN_METHODS = {
|
|
| 140 |
"emulator_use": "none",
|
| 141 |
"aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
|
| 142 |
},
|
| 143 |
-
"gboed": {
|
| 144 |
-
"abbreviation": "GBOED",
|
| 145 |
-
"Method": "Goal-oriented Bayesian Optimal Experimental Design",
|
| 146 |
"parallelism": "parallel-interacting",
|
| 147 |
"update_type": "general",
|
| 148 |
"method_goal": "uq",
|
| 149 |
"emulator_use": "within-optimize",
|
| 150 |
-
"aliases": ["gboed", "boed"],
|
| 151 |
},
|
| 152 |
"lm-kgmm": {
|
| 153 |
"abbreviation": "LM-KGMM",
|
|
@@ -237,6 +237,33 @@ KNOWN_METHODS = {
|
|
| 237 |
"emulator_use": "none",
|
| 238 |
"aliases": ["rl-td3", "rl-td3-long"],
|
| 239 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
}
|
| 241 |
|
| 242 |
|
|
@@ -292,6 +319,41 @@ METHOD_COLORS: dict[str, str] = {
|
|
| 292 |
}
|
| 293 |
|
| 294 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
def build_alias_lookup() -> dict[str, str]:
|
| 296 |
lookup: dict[str, str] = {}
|
| 297 |
for canonical_name, meta in KNOWN_METHODS.items():
|
|
|
|
| 140 |
"emulator_use": "none",
|
| 141 |
"aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
|
| 142 |
},
|
| 143 |
+
"gboed-tmcmc": {
|
| 144 |
+
"abbreviation": "GBOED-TMCMC",
|
| 145 |
+
"Method": "Goal-oriented Bayesian Optimal Experimental Design (TMCMC)",
|
| 146 |
"parallelism": "parallel-interacting",
|
| 147 |
"update_type": "general",
|
| 148 |
"method_goal": "uq",
|
| 149 |
"emulator_use": "within-optimize",
|
| 150 |
+
"aliases": ["gboed", "gboed-tmcmc", "boed", "gboed-tmcmc-eig", "boed-tmcmc-eig", "tmcmc-eig"],
|
| 151 |
},
|
| 152 |
"lm-kgmm": {
|
| 153 |
"abbreviation": "LM-KGMM",
|
|
|
|
| 237 |
"emulator_use": "none",
|
| 238 |
"aliases": ["rl-td3", "rl-td3-long"],
|
| 239 |
},
|
| 240 |
+
"ies-iekf": {
|
| 241 |
+
"abbreviation": "IES-IEKF",
|
| 242 |
+
"Method": "Iterated Emulate-Sample (IEKF)",
|
| 243 |
+
"parallelism": "parallel-interacting",
|
| 244 |
+
"update_type": "general",
|
| 245 |
+
"method_goal": "uq",
|
| 246 |
+
"emulator_use": "within-optimize",
|
| 247 |
+
"aliases": ["ies-iekf", "boed-iekf"],
|
| 248 |
+
},
|
| 249 |
+
"gboed-iekf": {
|
| 250 |
+
"abbreviation": "GBOED-IEKF",
|
| 251 |
+
"Method": "Goal-oriented Bayesian Optimal Experimental Design (IEKF)",
|
| 252 |
+
"parallelism": "parallel-interacting",
|
| 253 |
+
"update_type": "general",
|
| 254 |
+
"method_goal": "uq",
|
| 255 |
+
"emulator_use": "within-optimize",
|
| 256 |
+
"aliases": ["gboed-iekf", "gboed-iekf-eig", "boed-iekf-eig"],
|
| 257 |
+
},
|
| 258 |
+
"ies-tmcmc": {
|
| 259 |
+
"abbreviation": "IES-TMCMC",
|
| 260 |
+
"Method": "Iterated Emulate-Sample (TMCMC)",
|
| 261 |
+
"parallelism": "parallel-interacting",
|
| 262 |
+
"update_type": "general",
|
| 263 |
+
"method_goal": "uq",
|
| 264 |
+
"emulator_use": "within-optimize",
|
| 265 |
+
"aliases": ["ies-tmcmc", "boed-tmcmc"],
|
| 266 |
+
},
|
| 267 |
}
|
| 268 |
|
| 269 |
|
|
|
|
| 319 |
}
|
| 320 |
|
| 321 |
|
| 322 |
+
# Method families used to group rows in the suitability tables (the online table in
|
| 323 |
+
# common/leaderboard.py and the LaTeX tables from offline-plots/suitability_tables.py).
|
| 324 |
+
# The idea is to put methods of similar construction — and so similar performance —
|
| 325 |
+
# next to each other. Dict order = family order; list order = row order within a
|
| 326 |
+
# family. A family only appears on a board if one of its methods has data there.
|
| 327 |
+
# Every method that can appear in a suitability table must be listed here.
|
| 328 |
+
SUITABILITY_FAMILIES: dict[str, list[str]] = {
|
| 329 |
+
"CES": ["CES-EKI-CONST", "CES-EKI-DMC", "CES-IEKF-CONST"],
|
| 330 |
+
"Kalman": ["ETKI", "IEKF", "TEKI", "UKI"],
|
| 331 |
+
"Gradient": ["LM", "ADAM"],
|
| 332 |
+
"GFDT": ["LM-DSM", "ADAM-DSM", "LM-KGMM", "ADAM-KGMM"],
|
| 333 |
+
"RL": ["RL-DDPG", "RL-TD3", "RL-DDPG-SHORT", "RL-TD3-SHORT"],
|
| 334 |
+
"IES": ["IES-IEKF", "IES-TMCMC"],
|
| 335 |
+
"GBOED": ["GBOED-IEKF", "GBOED-TMCMC"],
|
| 336 |
+
"General": ["ABC", "HM"],
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
# Spelled-out display names for the SUITABILITY_FAMILIES keys (suitability table,
|
| 340 |
+
# home-page key, and the grouped method selector on the leaderboards).
|
| 341 |
+
FAMILY_FULL_NAMES: dict[str, str] = {
|
| 342 |
+
"CES": "Calibrate, Emulate, Sample (CES)",
|
| 343 |
+
"Kalman": "Kalman",
|
| 344 |
+
"Gradient": "Gradient",
|
| 345 |
+
"GFDT": "Generalized Fluctuation-Dissipation (GFDT)",
|
| 346 |
+
"RL": "Reinforcement Learning (RL)",
|
| 347 |
+
"IES": "Iterated Emulate-Sample (IES)",
|
| 348 |
+
"GBOED": "Goal-oriented Bayesian Experimental Design (GBOED)",
|
| 349 |
+
"General": "General",
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def family_full_name(family: str) -> str:
|
| 354 |
+
return FAMILY_FULL_NAMES.get(family, family)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
def build_alias_lookup() -> dict[str, str]:
|
| 358 |
lookup: dict[str, str] = {}
|
| 359 |
for canonical_name, meta in KNOWN_METHODS.items():
|
src/data_store.py
CHANGED
|
@@ -65,6 +65,8 @@ DATASET_FILES = {
|
|
| 65 |
"etki_results/leaderboard_etki_l96_flux-force_2026-09-10.nc",
|
| 66 |
"iekf_results/leaderboard_iekf_l96_flux-force_2026-09-10.nc",
|
| 67 |
"UKI_results/uki_l96_nn_forcing_ensemble_results.nc",
|
|
|
|
|
|
|
| 68 |
"opt_kgmm_score_lm/leaderboard_sblm_kgmm_parallel_l96_flux-force_2026-09-02.nc",
|
| 69 |
"opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l96_flux-force_2026-08-12.nc",
|
| 70 |
"opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l96_flux-force_2026-09-11.nc",
|
|
@@ -129,7 +131,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
|
|
| 129 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l63_ensemble_results_2026-07-09_minimal.nc"),
|
| 130 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l63_ensemble_results_2026-07-17_minimal.nc"),
|
| 131 |
("hm", "history-matching-uq_results/history-matching_l63_ensemble_results_2026-09-15_minimal.nc"),
|
| 132 |
-
("gboed", "gboed-uq_results/
|
|
|
|
|
|
|
|
|
|
| 133 |
("abc", "abc/leaderboard_abc_l63_2026-09-10_minimal.nc"),
|
| 134 |
],
|
| 135 |
"L96": [
|
|
@@ -138,7 +143,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
|
|
| 138 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_ensemble_results_2026-07-09_minimal.nc"),
|
| 139 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_ensemble_results_2026-07-17_minimal.nc"),
|
| 140 |
("hm", "history-matching-uq_results/history-matching_l96_const-force_2026-09-15_minimal.nc"),
|
| 141 |
-
("gboed", "gboed-uq_results/
|
|
|
|
|
|
|
|
|
|
| 142 |
("abc", "abc/leaderboard_abc_l96_const-force_2026-09-10_minimal.nc"),
|
| 143 |
],
|
| 144 |
"L96_NN_FORCING": [
|
|
@@ -147,6 +155,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
|
|
| 147 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_nn_forcing_ensemble_results_2026-07-09_minimal.nc"),
|
| 148 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_nn_forcing_ensemble_results_2026-07-17_minimal.nc"),
|
| 149 |
("hm", "history-matching-uq_results/history-matching_l96_flux-force_2026-09-15_minimal.nc"),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
("abc", "abc/leaderboard_abc_l96_flux-force_2026-09-10_minimal.nc"),
|
| 151 |
],
|
| 152 |
"L96_SPATIAL_FORCING": [
|
|
@@ -155,7 +167,10 @@ UQ_BUDGET_FILES: dict[str, list[tuple[str, str]]] = {
|
|
| 155 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_spatial_forcing_ensemble_results_2026-07-09_minimal.nc"),
|
| 156 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_spatial_forcing_ensemble_results_2026-07-17_minimal.nc"),
|
| 157 |
("hm", "history-matching-uq_results/history-matching_l96_vec-force_2026-09-15_minimal.nc"),
|
| 158 |
-
("gboed", "gboed-uq_results/
|
|
|
|
|
|
|
|
|
|
| 159 |
("abc", "abc/leaderboard_abc_l96_vec-force_2026-09-10_minimal.nc"),
|
| 160 |
],
|
| 161 |
}
|
|
|
|
| 65 |
"etki_results/leaderboard_etki_l96_flux-force_2026-09-10.nc",
|
| 66 |
"iekf_results/leaderboard_iekf_l96_flux-force_2026-09-10.nc",
|
| 67 |
"UKI_results/uki_l96_nn_forcing_ensemble_results.nc",
|
| 68 |
+
"adam_results/leaderboard_adam_l96_flux-force_2026-10-21.nc",
|
| 69 |
+
"levenberg_marquardt_results/leaderboard_lm_l96_flux-force_2026-10-20.nc",
|
| 70 |
"opt_kgmm_score_lm/leaderboard_sblm_kgmm_parallel_l96_flux-force_2026-09-02.nc",
|
| 71 |
"opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l96_flux-force_2026-08-12.nc",
|
| 72 |
"opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l96_flux-force_2026-09-11.nc",
|
|
|
|
| 131 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l63_ensemble_results_2026-07-09_minimal.nc"),
|
| 132 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l63_ensemble_results_2026-07-17_minimal.nc"),
|
| 133 |
("hm", "history-matching-uq_results/history-matching_l63_ensemble_results_2026-09-15_minimal.nc"),
|
| 134 |
+
("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l63_ensemble_results_2026-10-16_minimal.nc"),
|
| 135 |
+
("ies-iekf", "gboed-uq_results/boed-iekf_l63_ensemble_results_2026-10-19_minimal.nc"),
|
| 136 |
+
("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l63_ensemble_results_2026-10-19_minimal.nc"),
|
| 137 |
+
("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l63_ensemble_results_2026-10-16_minimal.nc"),
|
| 138 |
("abc", "abc/leaderboard_abc_l63_2026-09-10_minimal.nc"),
|
| 139 |
],
|
| 140 |
"L96": [
|
|
|
|
| 143 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_ensemble_results_2026-07-09_minimal.nc"),
|
| 144 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_ensemble_results_2026-07-17_minimal.nc"),
|
| 145 |
("hm", "history-matching-uq_results/history-matching_l96_const-force_2026-09-15_minimal.nc"),
|
| 146 |
+
("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l96_const-force_2026-10-16_minimal.nc"),
|
| 147 |
+
("ies-iekf", "gboed-uq_results/boed-iekf_l96_const-force_2026-10-19_minimal.nc"),
|
| 148 |
+
("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l96_const-force_2026-10-19_minimal.nc"),
|
| 149 |
+
("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l96_const-force_2026-10-16_minimal.nc"),
|
| 150 |
("abc", "abc/leaderboard_abc_l96_const-force_2026-09-10_minimal.nc"),
|
| 151 |
],
|
| 152 |
"L96_NN_FORCING": [
|
|
|
|
| 155 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_nn_forcing_ensemble_results_2026-07-09_minimal.nc"),
|
| 156 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_nn_forcing_ensemble_results_2026-07-17_minimal.nc"),
|
| 157 |
("hm", "history-matching-uq_results/history-matching_l96_flux-force_2026-09-15_minimal.nc"),
|
| 158 |
+
("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l96_flux-force_2026-10-16_minimal.nc"),
|
| 159 |
+
("ies-iekf", "gboed-uq_results/boed-iekf_l96_flux-force_2026-10-19_minimal.nc"),
|
| 160 |
+
("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l96_flux-force_2026-10-19_minimal.nc"),
|
| 161 |
+
("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l96_flux-force_2026-10-16_minimal.nc"),
|
| 162 |
("abc", "abc/leaderboard_abc_l96_flux-force_2026-09-10_minimal.nc"),
|
| 163 |
],
|
| 164 |
"L96_SPATIAL_FORCING": [
|
|
|
|
| 167 |
("ces-eki-const", "ces-eki-const_results/ces-eki-const_l96_spatial_forcing_ensemble_results_2026-07-09_minimal.nc"),
|
| 168 |
("ces-iekf-const", "ces-iekf-const_results/ces-iekf-const_l96_spatial_forcing_ensemble_results_2026-07-17_minimal.nc"),
|
| 169 |
("hm", "history-matching-uq_results/history-matching_l96_vec-force_2026-09-15_minimal.nc"),
|
| 170 |
+
("gboed-tmcmc", "gboed-uq_results/boed-tmcmc-eig_l96_vec-force_2026-10-16_minimal.nc"),
|
| 171 |
+
("ies-iekf", "gboed-uq_results/boed-iekf_l96_vec-force_2026-10-19_minimal.nc"),
|
| 172 |
+
("gboed-iekf", "gboed-uq_results/boed-iekf-eig_l96_vec-force_2026-10-19_minimal.nc"),
|
| 173 |
+
("ies-tmcmc", "gboed-uq_results/boed-tmcmc_l96_vec-force_2026-10-16_minimal.nc"),
|
| 174 |
("abc", "abc/leaderboard_abc_l96_vec-force_2026-09-10_minimal.nc"),
|
| 175 |
],
|
| 176 |
}
|
src/pages/MethodDetails.py
CHANGED
|
@@ -307,7 +307,7 @@ method_meta = {
|
|
| 307 |
],
|
| 308 |
"notes": "**Empirically also gets stuck in local minima near the initial condition, regardless of the damping schedule** — the same failure mode as ADAM, unsurprising for any single-trajectory local optimizer on this multimodal, chaotic-dynamics calibration surface.",
|
| 309 |
},
|
| 310 |
-
"GBOED": {
|
| 311 |
"citation": "Holthuijzen et al. 2026",
|
| 312 |
"url": "https://arxiv.org/abs/2508.13071",
|
| 313 |
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — Goal-oriented BOED, Algorithm 2 of Holthuijzen et al. 2026.",
|
|
@@ -330,12 +330,18 @@ method_meta = {
|
|
| 330 |
"switches from greedy to joint optimization once a greedy step's cost exceeds a threshold."
|
| 331 |
),
|
| 332 |
"params": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
{"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
|
| 334 |
-
{"Parameter": "
|
| 335 |
{"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
|
|
|
|
| 336 |
{"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
|
| 337 |
],
|
| 338 |
-
"notes": "**
|
| 339 |
},
|
| 340 |
"LM-KGMM": {
|
| 341 |
"citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025",
|
|
@@ -622,6 +628,132 @@ method_meta = {
|
|
| 622 |
"targets on L96 flux-force or vec-force within the budget."
|
| 623 |
),
|
| 624 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
}
|
| 626 |
|
| 627 |
# KNOWN_METHODS is the source of truth for the abbreviation/full-name list,
|
|
|
|
| 307 |
],
|
| 308 |
"notes": "**Empirically also gets stuck in local minima near the initial condition, regardless of the damping schedule** — the same failure mode as ADAM, unsurprising for any single-trajectory local optimizer on this multimodal, chaotic-dynamics calibration surface.",
|
| 309 |
},
|
| 310 |
+
"GBOED-TMCMC": {
|
| 311 |
"citation": "Holthuijzen et al. 2026",
|
| 312 |
"url": "https://arxiv.org/abs/2508.13071",
|
| 313 |
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — Goal-oriented BOED, Algorithm 2 of Holthuijzen et al. 2026.",
|
|
|
|
| 330 |
"switches from greedy to joint optimization once a greedy step's cost exceeds a threshold."
|
| 331 |
),
|
| 332 |
"params": [
|
| 333 |
+
{"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"},
|
| 334 |
+
{"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"},
|
| 335 |
+
{"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"},
|
| 336 |
+
{"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"},
|
| 337 |
+
{"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"},
|
| 338 |
{"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
|
| 339 |
+
{"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"},
|
| 340 |
{"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
|
| 341 |
+
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
|
| 342 |
{"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
|
| 343 |
],
|
| 344 |
+
"notes": "**Both the Transitional MCMC (TMCMC) posterior sampling and the EIG batch optimization contribute to performance.** TMCMC needs a sufficiently large population (`n_posterior_samples`) to represent the posterior, particularly in higher dimensions; its cost grows roughly linearly with population size, and `tmcmc_burnin`/`tmcmc_thin` multiply the per-stage cost. The EIG objective uses only a small posterior subsample (`n_eig_posterior_samples`), as EIG costs ~n³ per output mode. **The inner EIG-maximization loop is a frequent source of problems (hitting the call-limit / outer-iteration guardrails, i.e. \"under-optimized\" batches), and every iteration slows down as the cumulative GP training set grows**, which is why the higher-dimensional L96 cases use a smaller `max_iters`.",
|
| 345 |
},
|
| 346 |
"LM-KGMM": {
|
| 347 |
"citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025",
|
|
|
|
| 628 |
"targets on L96 flux-force or vec-force within the budget."
|
| 629 |
),
|
| 630 |
},
|
| 631 |
+
"IES-IEKF": {
|
| 632 |
+
"citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025",
|
| 633 |
+
"url": "https://doi.org/10.1029/2024MS004713",
|
| 634 |
+
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `acquire_batch`), variant `:iekf` (leaderboard key `boed-iekf`).",
|
| 635 |
+
"math": (
|
| 636 |
+
"An iterated emulate-sample loop. Iteration 1 forward-evaluates an "
|
| 637 |
+
"`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later "
|
| 638 |
+
"iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; "
|
| 639 |
+
"(ii) draws an approximate posterior by running an IEKF (`GaussNewtonInversion`) on the GP-mean "
|
| 640 |
+
"forward map — no true forward evaluations — starting from 1000 LHS prior draws; (iii) takes the next "
|
| 641 |
+
"`N_ens` forward-model evaluation points as a random subsample of that IEKF ensemble. The batch is "
|
| 642 |
+
"forward-evaluated and appended to the training set, so each iteration emulates the forward model, "
|
| 643 |
+
"samples the posterior from the emulator, and spends its forward runs where that posterior has mass."
|
| 644 |
+
),
|
| 645 |
+
"guardrails": (
|
| 646 |
+
"The posterior sampler is a fixed-step IEKF (`DefaultScheduler(0.1)`, 30 steps, total pseudo-time "
|
| 647 |
+
"T = 3), so it is a finite-time approximate posterior rather than a converged one; the observation "
|
| 648 |
+
"noise in whitened output space is the identity and the GP predictive variance is ignored. GP fits "
|
| 649 |
+
"keep kernel and noise hyperparameters inside bounds set relative to each mode's data scale (±2 "
|
| 650 |
+
"decades on length scales and signal std; noise std between 1e-3 and 1 of the signal std) to avoid "
|
| 651 |
+
"degenerate near-zero-noise fits, with `Fminbox` capped at 50 inner / 3 outer iterations. Inputs "
|
| 652 |
+
"and outputs are PCA-whitened and truncated against the prior and the observation covariance."
|
| 653 |
+
),
|
| 654 |
+
"params": [
|
| 655 |
+
{"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"},
|
| 656 |
+
{"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"},
|
| 657 |
+
{"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the IEKF ensemble)"},
|
| 658 |
+
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
|
| 659 |
+
{"Parameter": "retain_var (output whitening)", "Value": "0.99"},
|
| 660 |
+
{"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"},
|
| 661 |
+
],
|
| 662 |
+
"notes": (
|
| 663 |
+
"**Cheap per iteration:** no MCMC and no batch optimization, only GP-mean predictions "
|
| 664 |
+
"inside the IEKF. The batch is a plain draw from the posterior with no diversity criterion, so points are "
|
| 665 |
+
"not spread out deliberately. In the current UQ data (c = 1.5, best ensemble size) it needs a "
|
| 666 |
+
"mean budget of 5.0 on L63 and 5.4 on L96 const-force, but on the two high-dimensional L96 cases "
|
| 667 |
+
"(vec-force and NN flux-force) it fails on at least 35% and 40% of seeds, respectively, at every ensemble size."
|
| 668 |
+
),
|
| 669 |
+
},
|
| 670 |
+
"GBOED-IEKF": {
|
| 671 |
+
"citation": "Holthuijzen et al. 2026 (EIG acquisition); IEKF posterior sampler replaces ST-MCMC",
|
| 672 |
+
"url": "https://arxiv.org/abs/2508.13071",
|
| 673 |
+
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `optimize_batch`, `acquire_batch`), variant `:iekf_eig` (leaderboard key `boed-iekf-eig`).",
|
| 674 |
+
"math": (
|
| 675 |
+
"GBOED with the posterior sampler swapped for an IEKF. Iteration 1 forward-evaluates an `N_ens`-point "
|
| 676 |
+
"Latin-hypercube design in the truncated prior-whitened space. Each later iteration: (i) refits one GP "
|
| 677 |
+
"per whitened output mode on all `(θ, G(θ))` pairs so far; (ii) draws an approximate posterior by "
|
| 678 |
+
"running an IEKF (`GaussNewtonInversion`) on the GP-mean forward map — no true forward evaluations; "
|
| 679 |
+
"(iii) chooses the next `N_ens` evaluation points by jointly maximizing the closed-form Expected "
|
| 680 |
+
"Information Gain (EIG) of the whole batch against a 50-member random subsample X' of the IEKF "
|
| 681 |
+
"ensemble, starting from a random subsample of the full ensemble."
|
| 682 |
+
),
|
| 683 |
+
"math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}",
|
| 684 |
+
"guardrails": (
|
| 685 |
+
"The IEKF posterior is a fixed-step, finite-time approximation (`DefaultScheduler(0.1)`, 30 steps, "
|
| 686 |
+
"T = 3) that ignores the GP predictive variance. The batch is optimized with `Optim.jl`'s "
|
| 687 |
+
"`LBFGS` inside the χ² ball of the prior-whitened space (radial `tanh` squash, so every candidate "
|
| 688 |
+
"has radius below √χ²_k(0.999)), with a hard cap on raw EIG evaluations (`eig_call_limit`) and an "
|
| 689 |
+
"outer-iteration cap (`eig_outer_iters`) enforced manually, logging a warning that the returned "
|
| 690 |
+
"batch may be \"under-optimized\" if either is hit; a diagonal jitter (`eig_jitter`, relative to the "
|
| 691 |
+
"GP signal variance) stabilizes the covariance solves and log-determinants. GP fits use the same "
|
| 692 |
+
"data-relative hyperparameter bounds and iteration caps as IES-IEKF."
|
| 693 |
+
),
|
| 694 |
+
"params": [
|
| 695 |
+
{"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"},
|
| 696 |
+
{"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"},
|
| 697 |
+
{"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"},
|
| 698 |
+
{"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"},
|
| 699 |
+
{"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"},
|
| 700 |
+
{"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
|
| 701 |
+
{"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"},
|
| 702 |
+
{"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
|
| 703 |
+
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
|
| 704 |
+
{"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
|
| 705 |
+
{"Parameter": "retain_var / retain_var_input", "Value": "0.99 / 0.99 (L63, L96 const), 0.9 (vec, flux)"},
|
| 706 |
+
],
|
| 707 |
+
"notes": (
|
| 708 |
+
"**Adds the EIG batch optimization to IES-IEKF while keeping the cheap IEKF posterior**, so it avoids "
|
| 709 |
+
"ST-MCMC cost but still pays for the inner EIG-maximization loop, which costs ~n³ per output mode in "
|
| 710 |
+
"the subsample size and slows every iteration as the cumulative GP training set grows; hitting its "
|
| 711 |
+
"call or outer-iteration limits gives \"under-optimized\" batches. In the current UQ data (c = 1.5, "
|
| 712 |
+
"best ensemble size) it has the lowest mean budget of the four variants on L63 (4.8) and ties "
|
| 713 |
+
"GBOED-TMCMC on L96 const-force (4.4). On the two high-dimensional L96 cases it has low budgets "
|
| 714 |
+
"(55 on vec-force, 50 on NN flux-force) but fails on at least 35% and 40% of seeds, respectively, "
|
| 715 |
+
"at every ensemble size, so it is not suitable there under the 20% failure threshold."
|
| 716 |
+
),
|
| 717 |
+
},
|
| 718 |
+
"IES-TMCMC": {
|
| 719 |
+
"citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025",
|
| 720 |
+
"url": "https://doi.org/10.1029/2024MS004713",
|
| 721 |
+
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_tmcmc`, `boed_loglik`, `acquire_batch`), variant `:tmcmc` (leaderboard key `boed-tmcmc`).",
|
| 722 |
+
"math": (
|
| 723 |
+
"An iterated emulate-sample loop. Iteration 1 forward-evaluates an "
|
| 724 |
+
"`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later "
|
| 725 |
+
"iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; "
|
| 726 |
+
"(ii) draws posterior samples from the GP-based posterior via Transitional/Sequential MCMC "
|
| 727 |
+
"(Ching & Chen, 2007), using the GP-marginal Gaussian likelihood below; (iii) takes the next `N_ens` "
|
| 728 |
+
"forward-model evaluation points as a random subsample of those posterior draws. The batch is "
|
| 729 |
+
"forward-evaluated and appended to the training set, so each iteration emulates the forward model, "
|
| 730 |
+
"samples the posterior from the emulator, and spends its forward runs where that posterior has mass."
|
| 731 |
+
),
|
| 732 |
+
"math_eq": r"\log p(y\mid z) = \sum_j \left[-\frac{(\mu_j(z)-y_j)^2}{2\,(1+\sigma_j^2(z))} - \frac{1}{2}\log\left(2\pi\,(1+\sigma_j^2(z))\right)\right]",
|
| 733 |
+
"guardrails": (
|
| 734 |
+
"The likelihood includes the GP predictive variance `σ_j²(z)` (the whitened observation noise is the "
|
| 735 |
+
"identity), so the posterior widens where the surrogate is uncertain, and keeps the normalizing "
|
| 736 |
+
"log(2π·v) term because ST-MCMC needs a properly normalized density. TMCMC runs with burn-in and "
|
| 737 |
+
"thinning below its own 20 / 3 defaults since they multiply per-stage cost. GP fits keep "
|
| 738 |
+
"hyperparameters inside data-relative bounds (±2 decades on length scales and signal std; noise std "
|
| 739 |
+
"between 1e-3 and 1 of the signal std) to avoid the near-zero-noise fits that ST-MCMC resampling "
|
| 740 |
+
"can collapse onto, with `Fminbox` capped at 50 inner / 3 outer iterations."
|
| 741 |
+
),
|
| 742 |
+
"params": [
|
| 743 |
+
{"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"},
|
| 744 |
+
{"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"},
|
| 745 |
+
{"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the TMCMC draws)"},
|
| 746 |
+
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
|
| 747 |
+
{"Parameter": "retain_var (output whitening)", "Value": "0.99"},
|
| 748 |
+
{"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"},
|
| 749 |
+
],
|
| 750 |
+
"notes": (
|
| 751 |
+
"**The batch is a plain draw from the ST-MCMC posterior, with no batch optimization.** TMCMC cost grows roughly linearly with "
|
| 752 |
+
"the population size, and a small population gives a degenerate posterior in the high-dimensional "
|
| 753 |
+
"L96 cases. In the current UQ data (c = 1.5, best ensemble size) its mean budget is 6.4 on L63 and 5.6 on L96 "
|
| 754 |
+
"const-force, but on L96 vec-force it is 240 and it fails on at least 85% of seeds at every ensemble size."
|
| 755 |
+
),
|
| 756 |
+
},
|
| 757 |
}
|
| 758 |
|
| 759 |
# KNOWN_METHODS is the source of truth for the abbreviation/full-name list,
|
src/pages/UQLeaderboard.py
CHANGED
|
@@ -35,7 +35,6 @@ render_leaderboard(
|
|
| 35 |
state_prefix="uq",
|
| 36 |
default_target=1.5,
|
| 37 |
show_failure_panel=True,
|
| 38 |
-
show_scoring_modes=False,
|
| 39 |
canonical_target_levels=UQ_TARGET_LEVELS,
|
| 40 |
budget_store=load_uq_budget_store(),
|
| 41 |
benchmark_dims=BENCHMARK_DIMS,
|
|
|
|
| 35 |
state_prefix="uq",
|
| 36 |
default_target=1.5,
|
| 37 |
show_failure_panel=True,
|
|
|
|
| 38 |
canonical_target_levels=UQ_TARGET_LEVELS,
|
| 39 |
budget_store=load_uq_budget_store(),
|
| 40 |
benchmark_dims=BENCHMARK_DIMS,
|
src/streamlit_app.py
CHANGED
|
@@ -5,11 +5,11 @@ import pandas as pd
|
|
| 5 |
import streamlit as st
|
| 6 |
|
| 7 |
try:
|
| 8 |
-
from common.method_registry import KNOWN_METHODS
|
| 9 |
from common.leaderboard import _UPDATE_TYPE_ORDER
|
| 10 |
except ModuleNotFoundError:
|
| 11 |
sys.path.append(str(Path(__file__).resolve().parent))
|
| 12 |
-
from common.method_registry import KNOWN_METHODS
|
| 13 |
from common.leaderboard import _UPDATE_TYPE_ORDER
|
| 14 |
|
| 15 |
st.set_page_config(page_title="Calibration Benchmark", page_icon="🏠", layout="wide")
|
|
@@ -156,6 +156,18 @@ st.caption(
|
|
| 156 |
"general), not whether a method is 'Bayesian'."
|
| 157 |
)
|
| 158 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
st.subheader("Key metric")
|
| 160 |
st.markdown(
|
| 161 |
"The reported metric is the **mean number of forward-model evaluations** required "
|
|
|
|
| 5 |
import streamlit as st
|
| 6 |
|
| 7 |
try:
|
| 8 |
+
from common.method_registry import KNOWN_METHODS, SUITABILITY_FAMILIES, family_full_name
|
| 9 |
from common.leaderboard import _UPDATE_TYPE_ORDER
|
| 10 |
except ModuleNotFoundError:
|
| 11 |
sys.path.append(str(Path(__file__).resolve().parent))
|
| 12 |
+
from common.method_registry import KNOWN_METHODS, SUITABILITY_FAMILIES, family_full_name
|
| 13 |
from common.leaderboard import _UPDATE_TYPE_ORDER
|
| 14 |
|
| 15 |
st.set_page_config(page_title="Calibration Benchmark", page_icon="🏠", layout="wide")
|
|
|
|
| 156 |
"general), not whether a method is 'Bayesian'."
|
| 157 |
)
|
| 158 |
|
| 159 |
+
_family_key_rows = [
|
| 160 |
+
{"Family": family_full_name(fam), "Algorithms": ", ".join(members)}
|
| 161 |
+
for fam, members in SUITABILITY_FAMILIES.items()
|
| 162 |
+
]
|
| 163 |
+
st.markdown("**Method families** — algorithms of similar construction, used to group the suitability tables and method selectors:")
|
| 164 |
+
st.dataframe(
|
| 165 |
+
pd.DataFrame(_family_key_rows),
|
| 166 |
+
hide_index=True,
|
| 167 |
+
use_container_width=True,
|
| 168 |
+
height=int((len(_family_key_rows) + 1) * 35.2 + 3),
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
st.subheader("Key metric")
|
| 172 |
st.markdown(
|
| 173 |
"The reported metric is the **mean number of forward-model evaluations** required "
|