| from pathlib import Path |
| import sys |
|
|
| import pandas as pd |
| import streamlit as st |
|
|
| try: |
| from common.method_registry import KNOWN_METHODS |
| except ModuleNotFoundError: |
| sys.path.append(str(Path(__file__).resolve().parents[1])) |
| from common.method_registry import KNOWN_METHODS |
|
|
| st.set_page_config(page_title="Method Configurations", page_icon="📘", layout="wide") |
|
|
| |
| st.sidebar.title("Navigation") |
| st.sidebar.page_link("streamlit_app.py", label="Home", icon="🏠") |
| st.sidebar.page_link("pages/OptimizationLeaderboard.py", label="Optimization Leaderboard", icon="📊") |
| st.sidebar.page_link("pages/UQLeaderboard.py", label="UQ Leaderboard", icon="🎯") |
| st.sidebar.page_link("pages/MethodDetails.py", label="Method configurations", icon="📘") |
| st.sidebar.page_link("pages/RawData.py", label="Get Data", icon="🧾") |
|
|
| st.title("Method Configurations") |
| st.caption("Technical configuration of each calibration / UQ method, as implemented for this benchmark") |
| st.info( |
| "We are always seeking new algorithm families to be applied to this benchmark series. " |
| "If your favorite method is missing, or incorrectly configured, or if any algorithm is " |
| "lacking appropriate citation - please create an issue on `CalibrationLeaderboard.jl`, " |
| "and we can investigate this! Just note - we are interested in general behavior of " |
| "algorithm families, not hyper-optimizing any individual configuration to an individual " |
| "experiment." |
| ) |
| st.markdown( |
| "For the exact implementation of each algorithm, see " |
| "[CalibrationLeaderboard.jl](https://github.com/odunbar/CalibrationLeaderboard.jl) " |
| "(`opt_experiments/` and `uq_experiments/`)." |
| ) |
|
|
| |
| |
| |
| |
| |
| method_meta = { |
| "TEKI": { |
| "citation": "Chada et al., SIAM/ASA J. UQ, 2020", |
| "url": "https://doi.org/10.1137/17M114402X", |
| "source": "`opt_experiments/ensemble_kalman_processes` — EnsembleKalmanProcesses.jl `Inversion(prior)`.", |
| "math": ( |
| "Ensemble Kalman Inversion (Schillings & Stuart, 2017, eqns. 4-5): each iteration moves every " |
| "ensemble member by the same linear, empirical-covariance-based correction toward the data," |
| ), |
| "math_eq": r"u_{n+1} = u_n + C^{ug}\left(C^{gg}+\Gamma\right)^{-1}\left(y - g(u_n)\right)", |
| "guardrails": ( |
| "Tikhonov-style regularization is obtained by extending the state: the forward map is " |
| "augmented as `g ↦ [g; u]` and the data as `y ↦ [y; prior_mean]` with observation covariance " |
| "`blockdiag(Γ, prior_cov)`, so every step also pulls the ensemble back toward the prior " |
| "(`impose_prior=true`). A small `default_multiplicative_inflation` (1e-3) regularizes the " |
| "empirical covariance before it is inverted; `NoLocalization()` applies no covariance tapering; " |
| "`DefaultScheduler()` takes a fixed unit pseudo-time step each iteration." |
| ), |
| "params": [ |
| {"Parameter": "impose_prior", "Value": "true (Tikhonov-style state extension)"}, |
| {"Parameter": "default_multiplicative_inflation", "Value": "1e-3"}, |
| {"Parameter": "localization", "Value": "NoLocalization()"}, |
| {"Parameter": "scheduler", "Value": "DefaultScheduler() — fixed unit step"}, |
| ], |
| "notes": "Runs until the whitened RMSE falls below the target or `N_iter` is exhausted; no reported pathology on this benchmark beyond the usual small-ensemble sampling error of stochastic EKI.", |
| }, |
| "ETKI": { |
| "citation": "Schillings & Stuart, Numer. Math., 2017", |
| "url": "https://clima.github.io/EnsembleKalmanProcesses.jl/dev/", |
| "source": "`opt_experiments/ensemble_kalman_processes` — EnsembleKalmanProcesses.jl `TransformInversion(prior)`.", |
| "math": ( |
| "Ensemble Transform Kalman Inversion applies the update in ensemble-anomaly space instead of " |
| "perturbing observations: normalized anomalies `X=(u-ū)/√(m-1)` and `Y=(g_ext-ḡ_ext)/√(m-1)` are " |
| "formed (with the same prior extension as TEKI), and a transform matrix built from `Y'Γ⁻¹Y` maps " |
| "the prior ensemble anomalies onto posterior anomalies, `u ← ū + X·T`." |
| ), |
| "guardrails": ( |
| "Same Tikhonov-style prior extension and `impose_prior=true` as TEKI; diagonal regularization " |
| "stabilizes the anomaly Gram matrix before it is inverted; `NoLocalization()`; the transform " |
| "update removes the perturbed-observation sampling noise present in stochastic EKI, which " |
| "generally lowers ensemble-collapse risk at small ensemble sizes." |
| ), |
| "params": [ |
| {"Parameter": "impose_prior", "Value": "true (Tikhonov-style state extension)"}, |
| {"Parameter": "localization", "Value": "NoLocalization()"}, |
| {"Parameter": "scheduler", "Value": "DefaultScheduler() — fixed unit step"}, |
| ], |
| "notes": "Same run-until-RMSE-target harness as TEKI; the transform (square-root) update trades stochastic sampling noise for a deterministic ensemble-anomaly map.", |
| }, |
| "IEKF": { |
| "citation": "Iglesias, Inverse Problems, 2016", |
| "url": "https://doi.org/10.1088/0266-5611/32/2/025002", |
| "source": "`uq_experiments/GaussNewtonKalmanInversion` — EnsembleKalmanProcesses.jl `GaussNewtonInversion(prior)` (alias GNKI); Chada, Chen & Sanz-Alonso, 2021, Algorithm 4.1.", |
| "math": ( |
| "Iterated Ensemble Kalman Filter with statistical linearization: each pseudo-time step `Δt` " |
| "relaxes the ensemble between its current position and a linearized Gauss-Newton solve of the " |
| "joint parameter/output covariance, evaluated against a fresh perturbed draw around the prior " |
| "mean rather than the ensemble's own mean — a Gauss-Newton normal-equation solve carried out " |
| "entirely in empirical-covariance space, without ever forming the model Jacobian explicitly." |
| ), |
| "guardrails": ( |
| "`perturbation_scale = 2/Δt - 1` rescales the prior-draw noise so the relaxation step's " |
| "stationary covariance equals the true posterior for every `Δt ∈ (0,1]`, not just `Δt=1`; " |
| "diagonal regularization + `NoLocalization()` stabilize the joint covariance solve; the pseudo-" |
| "step is fixed rather than adaptive (`DefaultScheduler(0.1)`, i.e. `Δt=0.1`)." |
| ), |
| "params": [ |
| {"Parameter": "pseudo-step Δt", "Value": "0.1 (DefaultScheduler(0.1))"}, |
| {"Parameter": "perturbation_scale", "Value": "2/Δt - 1 = 19"}, |
| {"Parameter": "localization", "Value": "NoLocalization()"}, |
| ], |
| "notes": "An iterative Gauss-Newton procedure, so — like other Newton-type ensemble methods — it can be sensitive to ensemble size at small N_ens; only the first `max_iter` (10 on L63, 15 on L96) of the `N_iter=20` allowed steps are scored.", |
| }, |
| "UKI": { |
| "citation": "Huang, Huang & Stuart, Physica D, 2022", |
| "url": "https://clima.github.io/EnsembleKalmanProcesses.jl/dev/", |
| "source": "`opt_experiments/ensemble_kalman_processes` — EnsembleKalmanProcesses.jl `Unscented(prior; impose_prior=true)`.", |
| "math": ( |
| "Unscented Kalman Inversion propagates `2·N_par + 1` deterministic sigma points (the symmetric " |
| "unscented transform) through the forward model instead of a random ensemble, forms their " |
| "weighted mean/covariance, and performs the same Kalman-style analysis step as TEKI on this " |
| "reduced sigma-point set." |
| ), |
| "guardrails": ( |
| "`α_reg ∈ (0,1]` shrinks the analysis mean toward the prior mean each step, guarding against " |
| "covariance collapse; `impose_prior=true` adds the same TEKI-style `y_ext`/`Γ_ext` extension; " |
| "the symmetric sigma-point scheme (`2·N_par+1` points) is used by default rather than the " |
| "cheaper simplex scheme." |
| ), |
| "params": [ |
| {"Parameter": "α_reg", "Value": "1.0 (regularization toward prior mean disabled by default)"}, |
| {"Parameter": "sigma_points", "Value": '"symmetric" (2·N_par+1 points)'}, |
| {"Parameter": "impose_prior", "Value": "true"}, |
| ], |
| "notes": "Sigma-point count is fixed by the parameter dimension, not swept as an ensemble size — UKI's per-iteration cost is dimension-driven rather than N_ens-driven.", |
| }, |
| "ABC": { |
| "citation": "Csilléry et al., Trends in Ecology & Evolution, 2010", |
| "url": "https://doi.org/10.1016/j.tree.2010.04.001", |
| "source": "`uq_experiments/approximate_bayesian_computation` — simulation-based rejection sampling (no EKP, no surrogate).", |
| "math": ( |
| "Each round draws `N_ens` i.i.d. prior candidates `θ ~ π(θ)`, evaluates the true forward model " |
| "`G(θ)`, and accepts a candidate iff its whitened misfit falls inside a chi-squared acceptance " |
| "region; accepted candidates across all rounds approximate draws from the ABC posterior." |
| ), |
| "math_eq": r"\left(G(\theta)-y\right)^{\top}(2R)^{-1}\left(G(\theta)-y\right) \le \chi^2_{n_y}(1-\alpha_{\text{reject}})", |
| "guardrails": ( |
| "The acceptance threshold is set exactly to the `(1-alpha_reject)` quantile of `χ²(n_y)` rather " |
| "than hand-tuned; the final pushforward resamples `n_pushforward_samples=1000` draws from a " |
| "Gaussian fit to the accepted pool, smoothing a possibly sparse accepted set into a usable " |
| "posterior approximation; batch sizes are deliberately matched to IEKF's `N_ens_sizes`/`max_iter` " |
| "so the N_ens×k budget axis is directly comparable across methods on the leaderboard." |
| ), |
| "params": [ |
| {"Parameter": "alpha_reject", "Value": "0.05"}, |
| {"Parameter": "n_pushforward_samples", "Value": "1000"}, |
| ], |
| "notes": "Acceptance rate collapses quickly in higher dimensions (the same \"curse of dimensionality\" documented for HM below); there is no adaptive tightening of the acceptance region as rounds proceed, so all cost-efficiency comes from the batch-size choice. The Gaussian fit to the accepted pool additionally requires at least `p+2` accepted posterior samples before it can be estimated (`p` = parameter dimension); in higher-dimensional experiments this minimum is never reached within the 10,000-evaluation cap, so the pushforward Gaussian is never fit.", |
| }, |
| "HM": { |
| "citation": "Williamson et al. 2013; King et al. 2025", |
| "url": "https://example.com/hm", |
| "source": "`uq_experiments/HistoryMatching` — custom (non-EKP) wave/rejection scheme.", |
| "math": ( |
| "History Matching runs a series of waves. At wave k, an independent GP emulator is fit per " |
| "whitened output mode on an accumulated training set: the current ensemble plus all training " |
| "points from historical waves that lie within the current NROY space (points that have been " |
| "ruled out are dropped, so the emulator concentrates its accuracy on the shrinking NROY region); " |
| "a candidate survives the wave iff " |
| "its implausibility (whitened, GP-predicted misfit) is below a chi-squared threshold, and the " |
| "surviving (\"Not Ruled Out Yet\", NROY) region is intersected across all waves run so far." |
| ), |
| "math_eq": r"I^2(\theta) = \sum_j \frac{\left(\mu_j(\theta)-y_j\right)^2}{1+\sigma_j^2(\theta)} < \chi^2_{k_R}(0.95)", |
| "guardrails": ( |
| "GP emulators use an automatic-relevance-determination squared-exponential kernel (`SEArd`), " |
| "refit every wave via `GaussianProcesses.optimize!`; NROY testing exploits monotonicity — a " |
| "candidate ruled out by an earlier wave is never re-tested by later waves — cutting GP-predict " |
| "cost from O(waves×batch) to O(surviving candidates); a Poisson/Garwood upper-confidence-bound " |
| "rule (`poisson_rate_upper_bound`, `early_stop_alpha=1e-6`) aborts a rejection-sampling sweep " |
| "early once even its best-case acceptance rate can no longer plausibly reach the wanted sample " |
| "count within the remaining budget, rather than burning the full `max_samples` budget on a " |
| "doomed sweep; inputs/outputs are PCA-whitened (`retain_var_input`/`retain_var`) before fitting." |
| ), |
| "params": [ |
| {"Parameter": "confidence (implausibility threshold)", "Value": "0.95 → quantile(Chisq(k_R), 0.95)"}, |
| {"Parameter": "n_candidate_batch", "Value": "2000 per wave"}, |
| {"Parameter": "retain_var (output PCA)", "Value": "0.99"}, |
| {"Parameter": "retain_var_input (input PCA)", "Value": "0.9"}, |
| {"Parameter": "early_stop_alpha", "Value": "1e-6"}, |
| ], |
| "notes": "Acceptance rate collapses toward zero after a couple of waves on higher-dimensional problems (the repo's own documented \"curse of dimensionality\"); the early-stop rule only changes how quickly a doomed sweep gives up, not the eventual pass/fail outcome.", |
| }, |
| "CES-EKI-DMC": { |
| "citation": "Cleary et al., J. Comput. Phys., 2021", |
| "url": "https://doi.org/10.1016/j.jcp.2020.109716", |
| "source": "`uq_experiments/calibrate_emulate_sample` — Calibrate-Emulate-Sample, calibrate stage: `Inversion()` + `DataMisfitController`.", |
| "math": ( |
| "Three-stage Calibrate-Emulate-Sample. Calibrate: run EKI to generate an ensemble of " |
| "`(θ, G(θ))` training pairs. Emulate: fit a random-feature surrogate `Ĝ(θ)` on those pairs. " |
| "Sample: draw from `π(θ|y) ∝ exp(-½‖y-Ĝ(θ)‖²_Γ)·π(θ)` against the cheap surrogate using " |
| "preconditioned Crank-Nicolson Metropolis-Hastings (pCN-MH), rather than the true forward model." |
| ), |
| "guardrails": ( |
| "Calibrate stage uses an adaptive `DataMisfitController(terminate_at=2.0)` schedule so the EKI " |
| "pseudo-time step grows automatically and training points concentrate near the posterior, " |
| "rather than requiring a hand-tuned fixed schedule (`terminate_at=2.0` approximates the " |
| "finite-time posterior); emulate stage keeps 99% of variance in a likelihood-informed PCA " |
| "encoding before fitting the `ScalarRandomFeatureInterface`; sample stage auto-tunes the pCN " |
| "step size (`optimize_stepsize`, 2,000-sample dry run from `init_stepsize=0.2`) before the full " |
| "100,000-sample chain with a 2,000-sample burn-in." |
| ), |
| "params": [ |
| {"Parameter": "scheduler (calibrate)", "Value": "DataMisfitController(terminate_at=2.0)"}, |
| {"Parameter": "retain_var (emulate PCA)", "Value": "0.99"}, |
| {"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"}, |
| {"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"}, |
| {"Parameter": "init_stepsize", "Value": "0.2 (auto-tuned before the full chain)"}, |
| ], |
| "notes": "The emulator is fit only once, after calibration finishes (\"after-optimize\"), so sampled-posterior quality depends entirely on how well the calibrate-stage ensemble has already covered the posterior region.", |
| }, |
| "CES-EKI-CONST": { |
| "citation": "Cleary et al., J. Comput. Phys., 2021", |
| "url": "https://doi.org/10.1016/j.jcp.2020.109716", |
| "source": "`uq_experiments/calibrate_emulate_sample` — Calibrate-Emulate-Sample, calibrate stage: `Inversion()` + fixed-step scheduler.", |
| "math": "Identical Calibrate-Emulate-Sample pipeline to CES-EKI-DMC (see that entry for the emulate/sample math).", |
| "guardrails": ( |
| "Calibrate stage uses a fixed-step `DefaultScheduler(0.1)` instead of the adaptive " |
| "`DataMisfitController`, trading adaptivity for a simpler, more predictable schedule; emulate " |
| "and sample stages are unchanged from CES-EKI-DMC (same PCA retention, feature counts, and " |
| "MCMC step-size tuning)." |
| ), |
| "params": [ |
| {"Parameter": "scheduler (calibrate)", "Value": "DefaultScheduler(0.1) — fixed step"}, |
| {"Parameter": "retain_var (emulate PCA)", "Value": "0.99"}, |
| {"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"}, |
| {"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"}, |
| ], |
| "notes": "Same after-optimize emulator caveat as CES-EKI-DMC; the fixed step trades the DMC variant's adaptive posterior-concentration for predictability.", |
| }, |
| "CES-IEKF-CONST": { |
| "citation": "Cleary et al., J. Comput. Phys., 2021; Iglesias, Inverse Problems, 2016", |
| "url": "https://doi.org/10.1016/j.jcp.2020.109716", |
| "source": "`uq_experiments/calibrate_emulate_sample` — Calibrate-Emulate-Sample, calibrate stage: `GaussNewtonInversion(prior)` + fixed-step scheduler.", |
| "math": "Identical Calibrate-Emulate-Sample pipeline to CES-EKI-DMC, but the calibrate stage runs the IEKF/GNKI Gauss-Newton update (see the IEKF entry) instead of plain EKI.", |
| "guardrails": ( |
| "Calibrate stage uses `GaussNewtonInversion(prior)` with a fixed pseudo-step `DefaultScheduler(0.1)`; " |
| "emulate and sample stages are unchanged from CES-EKI-DMC (same PCA retention, feature counts, " |
| "and MCMC step-size tuning)." |
| ), |
| "params": [ |
| {"Parameter": "scheduler (calibrate)", "Value": "DefaultScheduler(0.1) — fixed pseudo-step"}, |
| {"Parameter": "retain_var (emulate PCA)", "Value": "0.99"}, |
| {"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"}, |
| {"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"}, |
| ], |
| "notes": "Same after-optimize emulator caveat as CES-EKI-DMC, with the Gauss-Newton calibrate step's usual small-ensemble sensitivity (see IEKF).", |
| }, |
| "CES-EKS": { |
| "citation": "Cleary et al., J. Comput. Phys., 2021; Garbuno-Inigo, Nüsken and Reich, SIAM J. Appl. Dyn. Syst., 2020", |
| "url": "https://doi.org/10.1016/j.jcp.2020.109716", |
| "source": "`uq_experiments/calibrate_emulate_sample` — `CALIBRATE_METHOD=eks` (calibrate stage `Sampler(prior)` + `EKSStableScheduler()`), `DIM_REDUCTION=PCA` (leaderboard key `ces-eks-pca`).", |
| "math": ( |
| "Calibrate-Emulate-Sample whose calibrate stage runs the Ensemble Kalman Sampler (see the EKS entry) instead of " |
| "plain EKI. Calibrate: run the sampler to generate `(θ, G(θ))` training pairs; iteration `k` trains on " |
| "the pairs from the first `k` steps. Emulate: fit a scalar random-feature surrogate `Ĝ(θ)` on those " |
| "pairs. Sample: draw from `π(θ|y) ∝ exp(-½‖y-Ĝ(θ)‖²_Γ)·π(θ)` against the cheap surrogate using " |
| "preconditioned Crank-Nicolson Metropolis-Hastings (pCN-MH), rather than the true forward model." |
| ), |
| "guardrails": ( |
| "Calibrate stage: `Sampler(prior)` with the data-dependent `EKSStableScheduler()`. The random-feature " |
| "kernel has a `1e-6` diagonal nugget. The nullspace-noise injector is switched off for this variant, so directions dropped by the " |
| "input PCA are decoded at the prior mean rather than filled with prior noise (the experiment config " |
| "notes that injecting noise inflated the posterior on L96 flux-force, with output spread about 2x too " |
| "wide and no contraction with `k`). Sample stage auto-tunes the pCN step size (`optimize_stepsize`, " |
| "2,000-sample dry run) before the full chain." |
| ), |
| "params": [ |
| {"Parameter": "calibrate sampler / scheduler", "Value": "Sampler(prior) / EKSStableScheduler()"}, |
| {"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"}, |
| {"Parameter": "kernel nugget", "Value": "1e-6"}, |
| {"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"}, |
| {"Parameter": "init_stepsize", "Value": "0.2 (L63) / 0.01 (L96), auto-tuned before the full chain"}, |
| {"Parameter": "nullspace noise injector", "Value": "off (threshold Inf)"}, |
| ], |
| "notes": ( |
| "The emulator is fit once per iteration `k` on the sampler's first `k` steps, so quality depends on " |
| "how well the calibrate-stage ensemble covers the posterior region. In the current UQ data " |
| "(c = 1.5, best reliable ensemble size) the mean budget is 5.3 on L63, 6.0 on L96 const-force, 87.5 " |
| "on vec-force (0% failures) and 207 on NN flux-force, which fails on at least 5% of seeds at every " |
| "size (15% at the best one)." |
| ), |
| }, |
| "ADAM": { |
| "citation": "Kingma & Ba, ICLR, 2015", |
| "url": "https://doi.org/10.48550/arXiv.1412.6980", |
| "source": "`opt_experiments/adam` — custom (non-Flux) Adam implementation.", |
| "math": ( |
| "Adam gradient descent on the whitened least-squares loss `L(θ)=½‖R^{-1/2}(y-G(θ))‖²`: " |
| "first/second raw-gradient moments `m_t=β₁m_{t-1}+(1-β₁)g_t`, `v_t=β₂v_{t-1}+(1-β₂)g_t²` are " |
| "bias-corrected to `m̂_t, v̂_t` and applied as `θ_{t+1}=θ_t - α·m̂_t/(√v̂_t+ε)`. The gradient " |
| "`g_t` comes from a whitened Jacobian obtained via `ForwardDiff.jl` forward-mode automatic " |
| "differentiation through the Lorenz solver itself — not a surrogate." |
| ), |
| "guardrails": ( |
| "`N_ens` initial-condition perturbations are drawn and averaged into the gradient at every step " |
| "to reduce gradient noise from the chaotic forward model (`N_ens=1` here, i.e. a single-sample " |
| "stochastic gradient); beyond Adam's own adaptive per-parameter step scaling there is no line " |
| "search, trust region, or restart mechanism." |
| ), |
| "params": [ |
| {"Parameter": "α (step size)", "Value": "0.001"}, |
| {"Parameter": "β₁, β₂", "Value": "0.9, 0.999"}, |
| {"Parameter": "ε", "Value": "1e-8"}, |
| {"Parameter": "N_ens (IC perturbations)", "Value": "1"}, |
| ], |
| "notes": "**Empirically gets stuck in local minima close to the initial parameter draw, regardless of step size** — observed running this benchmark. Consistent with Adam having no mechanism (trust region, restart, ensemble diversity) to escape a bad basin once its gradient estimate vanishes there.", |
| }, |
| "LM": { |
| "citation": "Levenberg, 1944; Marquardt, 1963; Fletcher, 1971", |
| "url": "https://doi.org/10.1090/qam/10666", |
| "source": "`opt_experiments/levenberg_marquardt` — custom Levenberg-Marquardt implementation.", |
| "math": ( |
| "Damped Gauss-Newton: each step solves the damped normal equations for the parameter update " |
| "`Δθ`, using a whitened Jacobian/residual `(J̃, r̃)` obtained via `ForwardDiff.jl`, then accepts " |
| "the step only if the reduction ratio `ρ` (actual vs. predicted misfit decrease) is favorable." |
| ), |
| "math_eq": r"\left(\tilde J^{\top}\tilde J + \lambda\,\mathrm{diag}(d)\right)\Delta\theta = \tilde J^{\top}\tilde r", |
| "guardrails": ( |
| "Damping `λ` adapts every step: ×4 (capped at `λ_max`) on a rejected or non-finite step " |
| "(`ρ<0.25` or NaN), ÷3 (floored at `λ_min`) on an accepted step (`ρ>0.75`), interpolating between " |
| "gradient descent (large λ) and Gauss-Newton (small λ); a non-finite `Δθ` is handled gracefully " |
| "(`ρ=NaN → !isfinite(ρ) → λ↑`) rather than crashing the run." |
| ), |
| "params": [ |
| {"Parameter": "λ₀ (initial damping)", "Value": "1.0"}, |
| {"Parameter": "λ growth / shrink factor", "Value": "×4 (reject) / ÷3 (accept)"}, |
| {"Parameter": "λ bounds", "Value": "[1e-10, 1e8]"}, |
| {"Parameter": "N_ens (IC perturbations)", "Value": "1"}, |
| ], |
| "notes": "**Empirically also gets stuck in local minima near the initial condition, regardless of the damping schedule** — the same failure mode as ADAM, unsurprising for any single-trajectory local optimizer on this multimodal, chaotic-dynamics calibration surface.", |
| }, |
| "GBOED-TMCMC": { |
| "citation": "Holthuijzen et al. 2026", |
| "url": "https://arxiv.org/abs/2508.13071", |
| "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — Goal-oriented BOED, Algorithm 2 of Holthuijzen et al. 2026.", |
| "math": ( |
| "Each iteration: (i) fits a GP surrogate per output dimension to all `(θ, G(θ))` collected so " |
| "far, (ii) draws posterior samples from the current surrogate-based posterior via Transitional/" |
| "Sequential MCMC (Ching & Chen, 2007), (iii) selects the next batch of forward-model evaluations " |
| "by maximizing a closed-form Expected Information Gain (EIG) criterion targeted at the quantity " |
| "of interest, using the GP's predictive covariance." |
| ), |
| "math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}", |
| "guardrails": ( |
| "Candidate batches are optimized with `Optim.jl`'s `Fminbox(LBFGS())` under box constraints, " |
| "with a hard cap on raw EIG evaluations (`eig_call_limit`) and an outer-iteration cap " |
| "(`eig_outer_iters`) enforced manually — since Optim's own call limits don't cumulate across " |
| "Fminbox's outer barrier rounds — logging a warning that the returned point may be " |
| "\"under-optimized\" if either cap is hit, rather than failing silently; a diagonal jitter " |
| "(`eig_jitter`, relative to the GP signal variance) stabilizes the covariance solves/log-" |
| "determinants; batch points can be chosen greedily, jointly, or via a hybrid strategy that " |
| "switches from greedy to joint optimization once a greedy step's cost exceeds a threshold." |
| ), |
| "params": [ |
| {"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"}, |
| {"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"}, |
| {"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"}, |
| {"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"}, |
| {"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"}, |
| {"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"}, |
| {"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"}, |
| {"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"}, |
| {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"}, |
| {"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"}, |
| ], |
| "notes": "**Both the Transitional MCMC (TMCMC) posterior sampling and the EIG batch optimization contribute to performance.** TMCMC needs a sufficiently large population (`n_posterior_samples`) to represent the posterior, particularly in higher dimensions; its cost grows roughly linearly with population size, and `tmcmc_burnin`/`tmcmc_thin` multiply the per-stage cost. The EIG objective uses only a small posterior subsample (`n_eig_posterior_samples`), as EIG costs ~n³ per output mode. **The inner EIG-maximization loop is a frequent source of problems (hitting the call-limit / outer-iteration guardrails, i.e. \"under-optimized\" batches), and every iteration slows down as the cumulative GP training set grows**, which is why the higher-dimensional L96 cases use a smaller `max_iters`.", |
| }, |
| "LM-KGMM": { |
| "citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025", |
| "url": "https://arxiv.org/abs/2503.18054", |
| "source": "`opt_experiments/score_based_lm` — Levenberg-Marquardt with a GFDT/KGMM Jacobian surrogate.", |
| "math": ( |
| "Same LM step as above, but the Jacobian is not differentiated through the solver — it is " |
| "estimated from trajectory data via the generalized fluctuation-dissipation theorem (GFDT): the " |
| "response of a fitted statistic `A(x)` to a parameter perturbation is approximated by a lag-" |
| "integrated, Tukey-tapered correlation with a \"conjugate\" variable `B(x)` built from the score " |
| "function `s(x)=∇log p(x)`, then chained through `dφ/dm` to the summary-statistic Jacobian." |
| ), |
| "math_eq": r"\frac{d\,\mathbb{E}[A]}{d\theta} \approx \int_0^{\tau_{\max}} \left\langle A(x_{t+\tau})\,B(x_t)\right\rangle\, w(\tau)\, d\tau", |
| "guardrails": ( |
| "The score `s(x)` is estimated with a k-means Gaussian-mixture score estimator (KGMM): the " |
| "trajectory is clustered into components whose mixture has a closed-form score. " |
| "`stein_recalibrate` rescales the estimated score so `-E[x·s(x)ᵀ]=I` holds on the calibration " |
| "ensemble (a built-in consistency check/correction); `kgmm_ridge` shrinks per-cluster " |
| "covariances toward the global covariance to avoid singular clusters; `is_collapsed_window` " |
| "(minimum trajectory variance below a tolerance) detects windows that have collapsed onto a " |
| "fixed point — GFDT is undefined there, since there is no invariant measure to differentiate — " |
| "and falls back to a direct `ForwardDiff.jl` Jacobian on those draws; total cost is capped at " |
| "`outer_iter × N_ens ≤ budget_total` per cell, trading ensemble size against iteration count " |
| "under a fixed evaluation budget." |
| ), |
| "params": [ |
| {"Parameter": "kgmm_ridge", "Value": "0.1"}, |
| {"Parameter": "tukey_alpha (response taper)", "Value": "0.75"}, |
| {"Parameter": "budget_total (outer_iter × N_ens)", "Value": "500"}, |
| {"Parameter": "collapse tolerance (min variance)", "Value": "1e-2"}, |
| ], |
| "notes": "**Much quicker to apply than LM-DSM** — fitting a k-means/Gaussian-mixture score is far cheaper than training a neural score network, so LM-KGMM stayed tractable on problems where LM-DSM's training cost made it impractical.", |
| }, |
| "LM-DSM": { |
| "citation": "Giorgini, Bischoff & Souza, arXiv:2509.19660, 2025", |
| "url": "https://arxiv.org/abs/2509.19660", |
| "source": "`opt_experiments/score_based_lm` — Levenberg-Marquardt with a GFDT/DSM Jacobian surrogate.", |
| "math": "Identical LM + GFDT scheme as LM-KGMM (see that entry's update equation), but the score `s(x)=∇log p(x)` is estimated by Denoising Score Matching (DSM) instead of KGMM.", |
| "guardrails": ( |
| "A neural network is trained to denoise samples perturbed by Gaussian noise at a single fixed " |
| "scale `σ`; its optimal denoiser equals the noised density's score (Vincent, 2011), and as " |
| "`σ→0` this recovers the true score at the cost of a harder training problem. `σ` is fixed per " |
| "benchmark rather than annealed, chosen empirically as the smallest noise level at which the " |
| "network still receives a usable learning signal. Same Stein recalibration and collapsed-window " |
| "`ForwardDiff.jl` fallback as LM-KGMM; same Tukey-tapered response integral and evaluation-" |
| "budget cap." |
| ), |
| "params": [ |
| {"Parameter": "σ (DSM noise scale)", "Value": "0.2 (L63) / 0.025 (L96 variants)"}, |
| {"Parameter": "tukey_alpha (response taper)", "Value": "0.75"}, |
| {"Parameter": "budget_total (outer_iter × N_ens)", "Value": "500"}, |
| {"Parameter": "collapse tolerance (min variance)", "Value": "1e-2"}, |
| ], |
| "notes": "**Training the DSM score network is very slow, even on moderately sized problems** — a 5-iteration smoke test on the higher-dimensional L96 forcing variants took ~39 minutes, extrapolating to several hours for a full run, and higher-dimensional experiments were killed after ~24h without finishing a sweep. Wall-clock cost, not accuracy, was the binding constraint on scaling LM-DSM to larger problems.", |
| }, |
| "ADAM-KGMM": { |
| "citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025", |
| "url": "https://arxiv.org/abs/2503.18054", |
| "source": "`opt_experiments/score_based_adam` — Adam with a GFDT/KGMM gradient surrogate.", |
| "math": ( |
| "Same Adam moment-based update as above, but the gradient `g_t` is not differentiated through " |
| "the solver — it is estimated from trajectory data via the generalized fluctuation-dissipation " |
| "theorem (GFDT), the same lag-integrated, Tukey-tapered response construction used by LM-KGMM " |
| "(see that entry's update equation), chained through `dφ/dm` to the whitened-loss gradient " |
| "instead of a Gauss-Newton Jacobian." |
| ), |
| "guardrails": ( |
| "Same GFDT machinery as LM-KGMM: the score `s(x)=∇log p(x)` is estimated with a k-means " |
| "Gaussian-mixture score estimator (KGMM), `stein_recalibrate` rescales it so `-E[x·s(x)ᵀ]=I` " |
| "holds on the calibration ensemble, `kgmm_ridge` shrinks per-cluster covariances toward the " |
| "global covariance, and `is_collapsed_window` falls back to a direct `ForwardDiff.jl` gradient " |
| "on windows that have collapsed onto a fixed point; total cost is capped at " |
| "`outer_iter × N_ens ≤ budget_total` per cell, same as LM-KGMM, trading ensemble size against " |
| "iteration count under a fixed evaluation budget; beyond that, Adam's own adaptive per-parameter " |
| "step scaling is the only guardrail on the step itself. Unlike `β₁`/`β₂`/`ε` (shared across all " |
| "cases), the step size `α` is set per-experiment from the case's prior — sized to ~0.25× the " |
| "mean prior standard deviation, so a run of consistent-sign steps crosses ~1 prior std in ~4 " |
| "outer iterations; a single global value tuned for one benchmark's residual scaling left Adam " |
| "unable to traverse even the L63/L96-const prior within its iteration budget." |
| ), |
| "params": [ |
| {"Parameter": "α (step size)", "Value": "Per-case, ~0.25× prior std (0.025–0.75 across benchmarks)"}, |
| {"Parameter": "β₁, β₂", "Value": "0.9, 0.999"}, |
| {"Parameter": "kgmm_ridge", "Value": "0.1"}, |
| {"Parameter": "tukey_alpha (response taper)", "Value": "0.75"}, |
| {"Parameter": "budget_total (outer_iter × N_ens)", "Value": "500"}, |
| ], |
| "notes": "**Inherits Adam's local-minimum susceptibility** (no trust region, restart, or ensemble diversity to escape a bad basin) **and LM-KGMM's cheaper-than-DSM score estimation** — the k-means/Gaussian-mixture score fit is far less costly than training a neural score network, so ADAM-KGMM stayed tractable on problems where a DSM-based Adam variant would not.", |
| }, |
| "ADAM-DSM": { |
| "citation": "Giorgini, Bischoff & Souza, arXiv:2509.19660, 2025", |
| "url": "https://arxiv.org/abs/2509.19660", |
| "source": "`opt_experiments/score_based_adam` — Adam with a GFDT/DSM gradient surrogate.", |
| "math": "Identical Adam + GFDT scheme as ADAM-KGMM (see that entry's update equation), but the score `s(x)=∇log p(x)` is estimated by Denoising Score Matching (DSM) instead of KGMM.", |
| "guardrails": ( |
| "Same DSM training scheme as LM-DSM: a neural network is trained to denoise samples perturbed " |
| "by Gaussian noise at a single fixed scale `σ`, whose optimal denoiser equals the noised " |
| "density's score (Vincent, 2011); `σ` is fixed per benchmark rather than annealed, chosen " |
| "empirically as the smallest noise level at which the network still receives a usable learning " |
| "signal. Same Stein recalibration and collapsed-window `ForwardDiff.jl` fallback, and the same " |
| "Tukey-tapered response integral and evaluation-budget cap as LM-DSM/ADAM-KGMM. Same per-case " |
| "step size `α` (shared `β₁`/`β₂`/`ε`) as ADAM-KGMM — sized to ~0.25× the case's prior standard " |
| "deviation, since a single global value tuned for one benchmark's residual scaling left Adam " |
| "unable to traverse even the L63/L96-const prior within its iteration budget." |
| ), |
| "params": [ |
| {"Parameter": "α (step size)", "Value": "Per-case, ~0.25× prior std (0.025–0.75 across benchmarks)"}, |
| {"Parameter": "β₁, β₂", "Value": "0.9, 0.999"}, |
| {"Parameter": "σ (DSM noise scale)", "Value": "0.2 (L63) / 0.025 (L96 variants)"}, |
| {"Parameter": "tukey_alpha (response taper)", "Value": "0.75"}, |
| {"Parameter": "budget_total (outer_iter × N_ens)", "Value": "500"}, |
| ], |
| "notes": "**Inherits both LM-DSM's slow DSM-training cost and Adam's local-minimum susceptibility** — training the DSM score network is the binding wall-clock cost (as with LM-DSM), and Adam's lack of a trust region or ensemble diversity leaves it as prone to getting stuck in a bad basin as ADAM-KGMM.", |
| }, |
| "RL-DDPG-SHORT": { |
| "citation": "Lillicrap et al., ICLR, 2016", |
| "url": "https://arxiv.org/abs/1509.02971", |
| "source": "`opt_experiments/ReinforcementLearning` — DDPG actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `UPDATE_FREQUENCY=short`.", |
| "math": ( |
| "The unknown forcing parameterization is the **policy** and the whitened statistics misfit " |
| "`Φ_k=‖y-G(x_{T_k})‖²_Γ` defines the **reward** `r_k=Φ_{k-1}-Φ_k`. A small critic MLP `q(x,F)` is " |
| "trained on replay-buffer Bellman targets `-(1-γ)Φ_k+γ·q(x_{k+1},π(x_{k+1}))` (the `Φ_{k-1}` " |
| "offset cancels exactly), and the policy parameters are ascended along `∇_F q` averaged over " |
| "buffer states with Adam. No Jacobian of the simulator is ever formed — the gradient comes " |
| "from differentiating the learned critic. **The prior enters in three ways, but not as a " |
| "penalty:** the initial policy is a random draw from the prior, the policy is stored in " |
| "prior-whitened coordinates `F=μ+Lθ̃` (`L` = Cholesky factor of the prior covariance) so the " |
| "Adam step and the Gaussian exploration noise are measured in prior standard deviations " |
| "(following the prior's correlations), and the critic sees the whitened action. The reward " |
| "is the misfit only. In the *short* variant one RL step advances a " |
| "sliding statistics window by 1/10 of its length, so there are ~10 RL updates per window of " |
| "simulated time, each costing a fraction of a forward evaluation." |
| ), |
| "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q_\vartheta(x,F)\right]_{F=F_\theta}\right)", |
| "guardrails": ( |
| "`γ=0.5<1` is essential: for a state-independent policy at `γ=1` the reward telescopes and " |
| "`∇_F Q` vanishes, so `γ=1` reproduces the no-learning control. A Polyak-averaged target critic " |
| "(`τ=0.05`) stabilizes the bootstrap; critic inputs/targets are standardized from warm-up data " |
| "and frozen; Gaussian exploration noise (0.1 prior std) is added to the applied action, with a " |
| "warm-up phase before the first update. The statistic is computed over a trailing window of " |
| "the full length `W` (not the short step) so it is not biased by a shortened estimation " |
| "window. Cost is counted as simulation only (1 initial full evaluation + stride/T per step); " |
| "per-cell budget is capped at 10000 units." |
| ), |
| "params": [ |
| {"Parameter": "γ (discount)", "Value": "0.5"}, |
| {"Parameter": "Polyak rate", "Value": "0.05"}, |
| {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"}, |
| {"Parameter": "batch size / updates per step", "Value": "64 / 1"}, |
| {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "10 / 50 / 5000"}, |
| {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"}, |
| {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"}, |
| ], |
| "notes": ( |
| "**Cost advantage is partly structural** — the continuing RL trajectory pays spin-up once rather " |
| "than every step (for L63 `T_start/T=0.75`), and the convergence test runs at every stride on " |
| "overlapping windows, so scores are not directly comparable to methods that re-pay spin-up each " |
| "evaluation. **Not tuned per case.** Converges on every seed on " |
| "L63 and L96-const, but the higher-dimensional forcings are the hard case for a learned " |
| "critic: on L96 flux-force and vec-force it reaches the targets on only a few of 30 seeds " |
| "(flux: 1 at target 1.1, 5 at 1.2; vec: 2 at 1.1, 4 at 1.2) and on none at 1.0 within the budget." |
| ), |
| }, |
| "RL-DDPG": { |
| "citation": "Lillicrap et al., ICLR, 2016", |
| "url": "https://arxiv.org/abs/1509.02971", |
| "source": "`opt_experiments/ReinforcementLearning` — DDPG actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `UPDATE_FREQUENCY=long` (leaderboard name `rl-ddpg-long`).", |
| "math": ( |
| "Same DDPG actor-critic scheme as RL-DDPG-SHORT (see that entry's update equation): the unknown " |
| "forcing parameterization is the **policy**, the decrease in the whitened statistics misfit " |
| "`Φ_k` is the **reward**, and the policy parameters are ascended along `∇_F q` of a small learned " |
| "critic. The prior enters as in RL-DDPG-SHORT: a random prior draw as the initial policy, " |
| "prior-whitened policy coordinates (step and exploration noise in prior standard deviations), " |
| "and no prior penalty in the reward. Here one RL step is the whole statistics window — exactly one standard forward " |
| "evaluation (spin-up + window), the same statistical window as every other method on the " |
| "leaderboard — so there is one RL update per forward evaluation and each step costs 1 unit." |
| ), |
| "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q_\vartheta(x,F)\right]_{F=F_\theta}\right)", |
| "guardrails": ( |
| "Same `γ=0.5<1` (at `γ=1` the critic has no `F`-dependence for a state-independent policy), " |
| "Polyak target critic (`τ=0.05`), frozen input/target standardization, and Gaussian exploration " |
| "noise (0.1 prior std) as RL-DDPG-SHORT. The statistic `G_k` is exactly the standard forward map " |
| "used by every other method, and the spin-up is paid on every step; per-cell budget is capped " |
| "at 10000 units." |
| ), |
| "params": [ |
| {"Parameter": "γ (discount)", "Value": "0.5"}, |
| {"Parameter": "Polyak rate", "Value": "0.05"}, |
| {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"}, |
| {"Parameter": "batch size / updates per step", "Value": "64 / 1"}, |
| {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "1 / 20 / 2000"}, |
| {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"}, |
| {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"}, |
| ], |
| "notes": ( |
| "**One RL update per forward evaluation, so far more expensive than RL-DDPG-SHORT** — the short " |
| "variant makes ~10× more updates per unit of cost and pays spin-up only once. **Not tuned per " |
| "case.** Converges reliably on L63 and L96-const, but never " |
| "reached the targets on L96 flux-force or vec-force within the budget." |
| ), |
| }, |
| "RL-TD3-SHORT": { |
| "citation": "Fujimoto, van Hoof & Meger, ICML, 2018", |
| "url": "https://arxiv.org/abs/1802.09477", |
| "source": "`opt_experiments/ReinforcementLearning` — TD3 actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `ALGORITHM=td3`, `UPDATE_FREQUENCY=short`.", |
| "math": ( |
| "Same actor-critic formulation as RL-DDPG-SHORT (see that entry): the unknown forcing " |
| "parameterization is the **policy**, the decrease in the whitened statistics misfit is the " |
| "**reward**, and the policy parameters are ascended along `∇_F q` of a learned critic. TD3 " |
| "replaces the single critic with **twin critics** whose minimum forms the Bellman target " |
| "`-(1-γ)Φ_k+γ·min_i q_i(x_{k+1},π(x_{k+1}))`, and the actor ascends critic 1 only. The prior " |
| "enters as in RL-DDPG-SHORT (random prior draw as the initial policy; prior-whitened policy " |
| "coordinates for the Adam step, exploration and target-smoothing noise; no prior penalty in " |
| "the reward). In the *short* variant one RL step advances a sliding statistics window by 1/10 of its length, so " |
| "there are ~10 RL updates per window of simulated time, each costing a fraction of a forward " |
| "evaluation." |
| ), |
| "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q^{(1)}_\vartheta(x,F)\right]_{F=F_\theta}\right)", |
| "guardrails": ( |
| "The three TD3 robustness heuristics are switched on relative to RL-DDPG: **twin critics** with a " |
| "min-target (reduces critic overestimation), **delayed policy updates** (the actor and the " |
| "Polyak target critics update once per 2 critic updates), and **target-policy smoothing** " |
| "(clipped Gaussian noise added to the target action, in prior-std units, since the actor is a " |
| "single vector). As for RL-DDPG there is no target actor. Same `γ=0.5<1`, Polyak rate, frozen " |
| "input/target standardization, 0.1 prior-std Gaussian exploration noise and trailing window of " |
| "full length `W` as RL-DDPG-SHORT; cost counts simulation only (1 initial evaluation + stride/T " |
| "per step), capped at 10000 units per cell." |
| ), |
| "params": [ |
| {"Parameter": "γ (discount)", "Value": "0.5"}, |
| {"Parameter": "Polyak rate", "Value": "0.05"}, |
| {"Parameter": "critics / policy delay", "Value": "2 / 2"}, |
| {"Parameter": "target smoothing noise (clip)", "Value": "0.2 (0.5), prior-std units"}, |
| {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"}, |
| {"Parameter": "batch size / updates per step", "Value": "64 / 1"}, |
| {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "10 / 50 / 5000"}, |
| {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"}, |
| {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"}, |
| ], |
| "notes": ( |
| "**Same structural cost advantage as RL-DDPG-SHORT** — the continuing trajectory pays spin-up " |
| "once and the convergence test runs on overlapping windows, so scores are not directly " |
| "comparable to methods that re-pay spin-up every evaluation. **Not tuned per case.** " |
| "Converges on every seed on L63 and L96-const (slower than RL-DDPG-SHORT on L63 at targets " |
| "1.0–1.1, similar at 1.2); on the higher-dimensional forcings it is weaker than RL-DDPG-SHORT: " |
| "it never reaches the targets on L96 flux-force, and on vec-force only 1 of 30 seeds does " |
| "(target 1.2) within the budget." |
| ), |
| }, |
| "RL-TD3": { |
| "citation": "Fujimoto, van Hoof & Meger, ICML, 2018", |
| "url": "https://arxiv.org/abs/1802.09477", |
| "source": "`opt_experiments/ReinforcementLearning` — TD3 actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `ALGORITHM=td3`, `UPDATE_FREQUENCY=long` (leaderboard name `rl-td3-long`).", |
| "math": ( |
| "Same TD3 actor-critic scheme as RL-TD3-SHORT (twin critics with a min-target; the actor " |
| "ascends critic 1 only; see that entry and RL-DDPG-SHORT for the formulation, including how " |
| "the prior sets the initial policy and the whitened coordinates). Here one RL step " |
| "is the whole statistics window — exactly one standard forward evaluation, the same statistical " |
| "window as every other method on the leaderboard — so there is one RL update per forward " |
| "evaluation and each step costs 1 unit." |
| ), |
| "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q^{(1)}_\vartheta(x,F)\right]_{F=F_\theta}\right)", |
| "guardrails": ( |
| "Same TD3 heuristics as RL-TD3-SHORT: twin critics with a min-target, delayed policy/target " |
| "updates (every 2 critic updates), and clipped target-policy smoothing noise. Same `γ=0.5<1`, " |
| "Polyak rate, frozen standardization and 0.1 prior-std exploration noise; the spin-up is paid " |
| "on every step; per-cell budget is capped at 10000 units." |
| ), |
| "params": [ |
| {"Parameter": "γ (discount)", "Value": "0.5"}, |
| {"Parameter": "Polyak rate", "Value": "0.05"}, |
| {"Parameter": "critics / policy delay", "Value": "2 / 2"}, |
| {"Parameter": "target smoothing noise (clip)", "Value": "0.2 (0.5), prior-std units"}, |
| {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"}, |
| {"Parameter": "batch size / updates per step", "Value": "64 / 1"}, |
| {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "1 / 20 / 2000"}, |
| {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"}, |
| {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"}, |
| ], |
| "notes": ( |
| "**One RL update per forward evaluation, so far more expensive than RL-TD3-SHORT.** **Not tuned " |
| "per case.** Converges on every seed on L63 and L96-const (slower than RL-DDPG " |
| "on L96-const, and on L63 faster at targets 1.0–1.1 but slower at 1.2), but never reached the " |
| "targets on L96 flux-force or vec-force within the budget." |
| ), |
| }, |
| "IES-IEKF": { |
| "citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025", |
| "url": "https://doi.org/10.1029/2024MS004713", |
| "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `acquire_batch`), variant `:iekf` (leaderboard key `boed-iekf`).", |
| "math": ( |
| "An iterated emulate-sample loop. Iteration 1 forward-evaluates an " |
| "`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later " |
| "iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; " |
| "(ii) draws an approximate posterior by running an IEKF (`GaussNewtonInversion`) on the GP-mean " |
| "forward map — no true forward evaluations — starting from 1000 LHS prior draws; (iii) takes the next " |
| "`N_ens` forward-model evaluation points as a random subsample of that IEKF ensemble. The batch is " |
| "forward-evaluated and appended to the training set, so each iteration emulates the forward model, " |
| "samples the posterior from the emulator, and spends its forward runs where that posterior has mass." |
| ), |
| "guardrails": ( |
| "The posterior sampler is a fixed-step IEKF (`DefaultScheduler(0.1)`, 30 steps, total pseudo-time " |
| "T = 3), so it is a finite-time approximate posterior rather than a converged one; the observation " |
| "noise in whitened output space is the identity and the GP predictive variance is ignored. GP fits " |
| "keep kernel and noise hyperparameters inside bounds set relative to each mode's data scale (±2 " |
| "decades on length scales and signal std; noise std between 1e-3 and 1 of the signal std) to avoid " |
| "degenerate near-zero-noise fits, with `Fminbox` capped at 50 inner / 3 outer iterations. Inputs " |
| "and outputs are PCA-whitened and truncated against the prior and the observation covariance." |
| ), |
| "params": [ |
| {"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"}, |
| {"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"}, |
| {"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the IEKF ensemble)"}, |
| {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"}, |
| {"Parameter": "retain_var (output whitening)", "Value": "0.99"}, |
| {"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"}, |
| ], |
| "notes": ( |
| "**Cheap per iteration:** no MCMC and no batch optimization, only GP-mean predictions " |
| "inside the IEKF. The batch is a plain draw from the posterior with no diversity criterion, so points are " |
| "not spread out deliberately. In the current UQ data (c = 1.5, best ensemble size) it needs a " |
| "mean budget of 5.0 on L63 and 5.4 on L96 const-force, but on the two high-dimensional L96 cases " |
| "(vec-force and NN flux-force) it fails on at least 35% and 40% of seeds, respectively, at every ensemble size." |
| ), |
| }, |
| "EKS": { |
| "citation": "Garbuno-Inigo, Nüsken and Reich, SIAM J. Appl. Dyn. Syst., 2020 (ALDI); Garbuno-Inigo, Hoffmann, Li and Stuart, SIAM J. Appl. Dyn. Syst., 2020 (original EKS)", |
| "url": "https://doi.org/10.1137/19M1304891", |
| "source": "`uq_experiments/KalmanSampler` — `calibrate_l63.jl`/`calibrate_l96.jl`, `experiment_config.jl` (`METHOD=sampler`, leaderboard key `sampler`); `EKP.Sampler(prior)` with EnsembleKalmanProcesses' default `sampler_type = \"aldi\"`.", |
| "math": ( |
| "EKS (a.k.a ALDI): the Ensemble Kalman Sampler in its affine-invariant Langevin form (ALDI). It is a " |
| "derivative-free interacting-particle system that approximately samples the posterior. Each iteration " |
| "moves every member with an ensemble-Kalman drift (parameter/output ensemble covariances stand in for " |
| "a simulator Jacobian), a prior term applied implicitly, a finite-sample correction `(d+1)/J` times " |
| "the member's deviation from the ensemble mean, and noise built from the ensemble square root. Unlike " |
| "EKI the noise keeps the ensemble from collapsing, so the ensemble approximates the posterior rather " |
| "than a point estimate. There is no emulator: the ensemble at each iteration is itself the posterior " |
| "estimate." |
| ), |
| "math_eq": r"\theta^{(j)}_{n+1} = \left(I + \Delta t_n C_n \Gamma_\theta^{-1}\right)^{-1}\left[\theta^{(j)}_n - \frac{\Delta t_n}{J}\sum_{k=1}^J \langle \mathcal{G}(\theta^{(k)}_n) - \bar{\mathcal{G}}_n,\, \Gamma_y^{-1}(\mathcal{G}(\theta^{(j)}_n) - y)\rangle (\theta^{(k)}_n - \bar\theta_n) + \Delta t_n C_n \Gamma_\theta^{-1} m_\theta + \Delta t_n \frac{d+1}{J}(\theta^{(j)}_n - \bar\theta_n)\right] + \sqrt{\frac{2\Delta t_n}{J}}\sum_{k=1}^J \xi_{jk}\,(\theta^{(k)}_n - \bar\theta_n)", |
| "guardrails": ( |
| "The step size comes from `EKSStableScheduler()` (numerator 1, nugget `eps()`), so it is adaptive and " |
| "data-dependent, `Δt = 1/(‖D‖ + eps)` with `D` the ensemble-weighted data-misfit matrix, and needs " |
| "no hand-tuned schedule." |
| ), |
| "params": [ |
| {"Parameter": "sampler_type", "Value": "\"aldi\" (EKP default)"}, |
| {"Parameter": "scheduler", "Value": "EKSStableScheduler() (numerator 1, nugget eps())"}, |
| ], |
| "notes": ( |
| "**No emulator and no MCMC; each iteration costs `N_ens` forward runs.** In the current UQ data " |
| "(c = 1.5, best reliable ensemble size) it needs a mean budget of 4.2 on L63 (15% failures) and 4.2 " |
| "on L96 const-force, taking only about one iteration. On the two high-dimensional cases it needs " |
| "7 to 8 iterations, so the budget grows with ensemble size: 353 at size 50 on NN flux-force and " |
| "460 at size 55 on vec-force (5% failures each); the largest sizes fail on up to 25-30% of seeds." |
| ), |
| }, |
| "IES-EKS": { |
| "citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025; Garbuno-Inigo, Hoffmann, Li and Stuart, SIAM J. Appl. Dyn. Syst., 2020 (EKS)", |
| "url": "https://doi.org/10.1029/2024MS004713", |
| "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_eks`, `acquire_batch`), variant `:eks` (leaderboard key `boed-eks`); `experiment_config.jl` sets `eks_sampler_type = \"eks\"`.", |
| "math": ( |
| "The same iterated emulate-sample loop as IES-IEKF and IES-TMCMC (see those entries), with the " |
| "posterior sampler swapped for the Ensemble Kalman Sampler (EKP `Sampler`). Iteration 1 " |
| "forward-evaluates an `N_ens`-point Latin-hypercube design in the truncated, prior-whitened space. " |
| "Each later iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs so far; " |
| "(ii) draws an approximate posterior by running the sampler on the GP-mean forward map (no true " |
| "forward evaluations), from a 200-member Latin-hypercube standard-normal start with the prior " |
| "N(0, I) and identity noise; (iii) takes the next `N_ens` forward-model evaluation points as a random " |
| "subsample of that ensemble." |
| ), |
| "guardrails": ( |
| "The sampler is `EKP.Sampler` with `sampler_type = \"eks\"` (the original EKS of Garbuno-Inigo, " |
| "Hoffmann, Li and Stuart, not the ALDI variant used by the EKS and CES-EKS entries), driven by " |
| "`EKSStableScheduler(alpha = 0.25)` for a fixed 500 steps. `alpha = 0.25` was chosen because it " |
| "sat closer to the ST-MCMC reference than the default 1, and `alpha >= 3` is unstable. Like the IEKF " |
| "variant it ignores the GP predictive variance. GP fits use the same data-relative hyperparameter " |
| "bounds and iteration caps as IES-IEKF. The posterior has 200 draws rather than 1000; the " |
| "shortfall is padded with NaN rows that the leaderboard stage drops." |
| ), |
| "params": [ |
| {"Parameter": "sampler_type", "Value": "\"eks\" (original EKS)"}, |
| {"Parameter": "EKS particles (eks_n_particles)", "Value": "200"}, |
| {"Parameter": "scheduler", "Value": "EKSStableScheduler(alpha = 0.25)"}, |
| {"Parameter": "steps (eks_max_iters / eks_T)", "Value": "500 / Inf (fixed step count)"}, |
| {"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the EKS ensemble)"}, |
| {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"}, |
| ], |
| "notes": ( |
| "**Cheap in forward runs but with a fixed 500-step sampler cost per iteration** (GP-mean evaluations " |
| "only). In the current UQ data (c = 1.5, best ensemble size) its mean budget is 5.4 on L63, 4.2 on " |
| "L96 const-force and 112 on NN flux-force (size 80), all with 0% failures. On vec-force it fails on " |
| "at least 20% of seeds at every ensemble size, so it is not suitable there under the 20% threshold." |
| ), |
| }, |
| "GBOED-EKS": { |
| "citation": "Holthuijzen et al. 2026 (EIG acquisition); Garbuno-Inigo, Hoffmann, Li and Stuart, SIAM J. Appl. Dyn. Syst., 2020 (EKS posterior sampler)", |
| "url": "https://arxiv.org/abs/2508.13071", |
| "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_eks`, `optimize_batch`, `acquire_batch`), variant `:eks_eig` (leaderboard key `boed-eks-eig`); `experiment_config.jl` sets `eks_sampler_type = \"eks\"`.", |
| "math": ( |
| "GBOED with the posterior sampler swapped for the Ensemble Kalman Sampler (see the GBOED-IEKF and " |
| "IES-EKS entries). Iteration 1 forward-evaluates an `N_ens`-point Latin-hypercube design in the " |
| "truncated prior-whitened space. Each later iteration: (i) refits one GP per whitened output mode on " |
| "all `(θ, G(θ))` pairs so far; (ii) draws an approximate posterior by running the sampler on the " |
| "GP-mean forward map (no true forward evaluations); (iii) chooses the next `N_ens` evaluation " |
| "points by jointly maximizing the closed-form Expected Information Gain (EIG) of the whole batch " |
| "against a 50-member random subsample of the EKS ensemble, starting from a random subsample of " |
| "the full ensemble." |
| ), |
| "math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}", |
| "guardrails": ( |
| "The sampler is `EKP.Sampler` with `sampler_type = \"eks\"` (original EKS, not ALDI), run for a " |
| "fixed 500 steps of `EKSStableScheduler(alpha = 0.25)` with 200 particles, ignoring the GP " |
| "predictive variance. The batch is optimized with `Optim.jl`'s `LBFGS` inside the χ² ball of the " |
| "prior-whitened space (radial `tanh` squash, so every candidate has radius below √χ²_k(0.999)), with " |
| "a hard cap on raw EIG evaluations (`eig_call_limit`) and an outer-iteration cap " |
| "(`eig_outer_iters`), logging a warning that the returned batch may be \"under-optimized\" if either " |
| "is hit; a diagonal jitter (`eig_jitter`, relative to the GP signal variance) stabilizes the " |
| "covariance solves and log-determinants. GP fits use the same data-relative bounds and iteration " |
| "caps as IES-IEKF." |
| ), |
| "params": [ |
| {"Parameter": "sampler_type", "Value": "\"eks\" (original EKS)"}, |
| {"Parameter": "EKS particles (eks_n_particles)", "Value": "200"}, |
| {"Parameter": "scheduler", "Value": "EKSStableScheduler(alpha = 0.25)"}, |
| {"Parameter": "steps (eks_max_iters / eks_T)", "Value": "500 / Inf (fixed step count)"}, |
| {"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"}, |
| {"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"}, |
| {"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"}, |
| {"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"}, |
| {"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"}, |
| {"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"}, |
| {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"}, |
| {"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"}, |
| ], |
| "notes": ( |
| "**Adds the EIG batch optimization to IES-EKS**, so it pays for the inner EIG-maximization loop on " |
| "top of the 500-step sampler; hitting its call or outer-iteration limits gives \"under-optimized\" " |
| "batches. In the current UQ data (c = 1.5, best ensemble size) its mean budget is 5.2 on L63, 4.2 on " |
| "L96 const-force and 105 on NN flux-force (size 80, 5% failures). On vec-force it fails on at least " |
| "25% of seeds at every ensemble size, so it is not suitable there under the 20% failure threshold." |
| ), |
| }, |
| "GBOED-IEKF": { |
| "citation": "Holthuijzen et al. 2026 (EIG acquisition); IEKF posterior sampler replaces ST-MCMC", |
| "url": "https://arxiv.org/abs/2508.13071", |
| "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `optimize_batch`, `acquire_batch`), variant `:iekf_eig` (leaderboard key `boed-iekf-eig`).", |
| "math": ( |
| "GBOED with the posterior sampler swapped for an IEKF. Iteration 1 forward-evaluates an `N_ens`-point " |
| "Latin-hypercube design in the truncated prior-whitened space. Each later iteration: (i) refits one GP " |
| "per whitened output mode on all `(θ, G(θ))` pairs so far; (ii) draws an approximate posterior by " |
| "running an IEKF (`GaussNewtonInversion`) on the GP-mean forward map — no true forward evaluations; " |
| "(iii) chooses the next `N_ens` evaluation points by jointly maximizing the closed-form Expected " |
| "Information Gain (EIG) of the whole batch against a 50-member random subsample X' of the IEKF " |
| "ensemble, starting from a random subsample of the full ensemble." |
| ), |
| "math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}", |
| "guardrails": ( |
| "The IEKF posterior is a fixed-step, finite-time approximation (`DefaultScheduler(0.1)`, 30 steps, " |
| "T = 3) that ignores the GP predictive variance. The batch is optimized with `Optim.jl`'s " |
| "`LBFGS` inside the χ² ball of the prior-whitened space (radial `tanh` squash, so every candidate " |
| "has radius below √χ²_k(0.999)), with a hard cap on raw EIG evaluations (`eig_call_limit`) and an " |
| "outer-iteration cap (`eig_outer_iters`) enforced manually, logging a warning that the returned " |
| "batch may be \"under-optimized\" if either is hit; a diagonal jitter (`eig_jitter`, relative to the " |
| "GP signal variance) stabilizes the covariance solves and log-determinants. GP fits use the same " |
| "data-relative hyperparameter bounds and iteration caps as IES-IEKF." |
| ), |
| "params": [ |
| {"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"}, |
| {"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"}, |
| {"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"}, |
| {"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"}, |
| {"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"}, |
| {"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"}, |
| {"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"}, |
| {"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"}, |
| {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"}, |
| {"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"}, |
| {"Parameter": "retain_var / retain_var_input", "Value": "0.99 / 0.99 (L63, L96 const), 0.9 (vec, flux)"}, |
| ], |
| "notes": ( |
| "**Adds the EIG batch optimization to IES-IEKF while keeping the cheap IEKF posterior**, so it avoids " |
| "ST-MCMC cost but still pays for the inner EIG-maximization loop, which costs ~n³ per output mode in " |
| "the subsample size and slows every iteration as the cumulative GP training set grows; hitting its " |
| "call or outer-iteration limits gives \"under-optimized\" batches. In the current UQ data (c = 1.5, " |
| "best ensemble size) it has the lowest mean budget of the four variants on L63 (4.8) and ties " |
| "GBOED-TMCMC on L96 const-force (4.4). On the two high-dimensional L96 cases it has low budgets " |
| "(55 on vec-force, 50 on NN flux-force) but fails on at least 35% and 40% of seeds, respectively, " |
| "at every ensemble size, so it is not suitable there under the 20% failure threshold." |
| ), |
| }, |
| "IES-TMCMC": { |
| "citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025", |
| "url": "https://doi.org/10.1029/2024MS004713", |
| "source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_tmcmc`, `boed_loglik`, `acquire_batch`), variant `:tmcmc` (leaderboard key `boed-tmcmc`).", |
| "math": ( |
| "An iterated emulate-sample loop. Iteration 1 forward-evaluates an " |
| "`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later " |
| "iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; " |
| "(ii) draws posterior samples from the GP-based posterior via Transitional/Sequential MCMC " |
| "(Ching & Chen, 2007), using the GP-marginal Gaussian likelihood below; (iii) takes the next `N_ens` " |
| "forward-model evaluation points as a random subsample of those posterior draws. The batch is " |
| "forward-evaluated and appended to the training set, so each iteration emulates the forward model, " |
| "samples the posterior from the emulator, and spends its forward runs where that posterior has mass." |
| ), |
| "math_eq": r"\log p(y\mid z) = \sum_j \left[-\frac{(\mu_j(z)-y_j)^2}{2\,(1+\sigma_j^2(z))} - \frac{1}{2}\log\left(2\pi\,(1+\sigma_j^2(z))\right)\right]", |
| "guardrails": ( |
| "The likelihood includes the GP predictive variance `σ_j²(z)` (the whitened observation noise is the " |
| "identity), so the posterior widens where the surrogate is uncertain, and keeps the normalizing " |
| "log(2π·v) term because ST-MCMC needs a properly normalized density. TMCMC runs with burn-in and " |
| "thinning below its own 20 / 3 defaults since they multiply per-stage cost. GP fits keep " |
| "hyperparameters inside data-relative bounds (±2 decades on length scales and signal std; noise std " |
| "between 1e-3 and 1 of the signal std) to avoid the near-zero-noise fits that ST-MCMC resampling " |
| "can collapse onto, with `Fminbox` capped at 50 inner / 3 outer iterations." |
| ), |
| "params": [ |
| {"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"}, |
| {"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"}, |
| {"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the TMCMC draws)"}, |
| {"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"}, |
| {"Parameter": "retain_var (output whitening)", "Value": "0.99"}, |
| {"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"}, |
| ], |
| "notes": ( |
| "**The batch is a plain draw from the ST-MCMC posterior, with no batch optimization.** TMCMC cost grows roughly linearly with " |
| "the population size, and a small population gives a degenerate posterior in the high-dimensional " |
| "L96 cases. In the current UQ data (c = 1.5, best ensemble size) its mean budget is 6.4 on L63 and 5.6 on L96 " |
| "const-force, but on L96 vec-force it is 240 and it fails on at least 85% of seeds at every ensemble size." |
| ), |
| }, |
| } |
|
|
| |
| |
| abbrs = sorted(meta["abbreviation"] for meta in KNOWN_METHODS.values()) |
| full_name_by_abbr = {meta["abbreviation"]: meta["Method"] for meta in KNOWN_METHODS.values()} |
|
|
| |
| abbr = st.query_params.get("method") |
| default_idx = abbrs.index(abbr) if isinstance(abbr, str) and abbr in abbrs else 0 |
| sel = str(st.selectbox("Choose a method", options=abbrs, index=default_idx)) |
|
|
| |
| st.query_params["method"] = sel |
|
|
| |
| meta = method_meta.get(sel, {}) |
|
|
| st.subheader(f"{sel} — {full_name_by_abbr.get(sel, '')}") |
| if meta.get("source"): |
| st.caption(meta["source"]) |
|
|
| st.markdown("**Update Equation**") |
| st.markdown(meta.get("math", "Mathematical form pending")) |
| if meta.get("math_eq"): |
| st.latex(meta["math_eq"]) |
|
|
| st.markdown("**Robustness, Speed & Guardrails**") |
| st.markdown(meta.get("guardrails", "Guardrail details pending")) |
|
|
| st.markdown("**Configuration Parameters**") |
| params = meta.get("params") |
| if params: |
| st.dataframe(pd.DataFrame(params), hide_index=True, use_container_width=True) |
| else: |
| st.write("Parameter details pending") |
|
|
| st.markdown("**Convergence & Speed Notes**") |
| st.markdown(meta.get("notes", "No notes recorded yet")) |
|
|
| col1, col2 = st.columns(2) |
| with col1: |
| with st.container(border=True): |
| st.markdown("**Citation**") |
| st.write(meta.get("citation", "Citation pending")) |
| with col2: |
| with st.container(border=True): |
| st.markdown("**Link**") |
| st.link_button("Open reference", meta.get("url", "https://example.com")) |
|
|
| st.page_link("pages/OptimizationLeaderboard.py", label="⬅ Back to Optimization Leaderboard", icon="↩️") |
|
|
