File size: 76,772 Bytes
fce6c09 d774735 ad36a58 fce6c09 d774735 fce6c09 d774735 fce6c09 d774735 fce6c09 43604b6 d774735 fce6c09 ad36a58 d774735 1ad9fc2 d774735 1ad9fc2 ad36a58 d774735 ad36a58 43604b6 ad36a58 d774735 ad36a58 fce6c09 43604b6 d774735 fce6c09 43604b6 d774735 fce6c09 ad36a58 43604b6 d774735 ad36a58 baf1286 d774735 ad11aa3 baf1286 43604b6 baf1286 d774735 92d135a d774735 baf1286 43604b6 d774735 43604b6 4bcf26d d774735 4bcf26d d774735 4bcf26d 3e13282 0817e0a d774735 0817e0a d774735 0817e0a 221faf8 eb6584e d774735 221faf8 d774735 221faf8 d774735 221faf8 d774735 221faf8 eb6584e 1ad9fc2 d774735 1ad9fc2 d774735 fd69d16 d774735 fd69d16 d774735 1ad9fc2 ad11aa3 fd69d16 ad11aa3 fd69d16 ad11aa3 763c770 fd69d16 763c770 fd69d16 763c770 fd69d16 763c770 fd69d16 763c770 1263fa8 221faf8 3e13282 fd69d16 3e13282 fd69d16 3e13282 fd69d16 3e13282 fd69d16 3e13282 fd69d16 3e13282 fd69d16 3e13282 fd69d16 3e13282 fd69d16 3e13282 221faf8 ad36a58 d774735 ad36a58 d774735 fce6c09 ad36a58 d774735 fce6c09 d774735 ad36a58 fce6c09 ad36a58 fce6c09 43604b6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 | from pathlib import Path
import sys
import pandas as pd
import streamlit as st
try:
from common.method_registry import KNOWN_METHODS
except ModuleNotFoundError:
sys.path.append(str(Path(__file__).resolve().parents[1]))
from common.method_registry import KNOWN_METHODS
st.set_page_config(page_title="Method Configurations", page_icon="📘", layout="wide")
# Sidebar navigation
st.sidebar.title("Navigation")
st.sidebar.page_link("streamlit_app.py", label="Home", icon="🏠")
st.sidebar.page_link("pages/OptimizationLeaderboard.py", label="Optimization Leaderboard", icon="📊")
st.sidebar.page_link("pages/UQLeaderboard.py", label="UQ Leaderboard", icon="🎯")
st.sidebar.page_link("pages/MethodDetails.py", label="Method configurations", icon="📘")
st.sidebar.page_link("pages/RawData.py", label="Get Data", icon="🧾")
st.title("Method Configurations")
st.caption("Technical configuration of each calibration / UQ method, as implemented for this benchmark")
st.info(
"We are always seeking new algorithm families to be applied to this benchmark series. "
"If your favorite method is missing, or incorrectly configured, or if any algorithm is "
"lacking appropriate citation - please create an issue on `CalibrationLeaderboard.jl`, "
"and we can investigate this! Just note - we are interested in general behavior of "
"algorithm families, not hyper-optimizing any individual configuration to an individual "
"experiment."
)
st.markdown(
"For the exact implementation of each algorithm, see "
"[CalibrationLeaderboard.jl](https://github.com/odunbar/CalibrationLeaderboard.jl) "
"(`opt_experiments/` and `uq_experiments/`)."
)
# Per-method metadata (can be moved to a JSON/YAML later). Every entry below was read
# directly out of CalibrationLeaderboard.jl (opt_experiments/, uq_experiments/) and, for the
# EKP-based processes, out of EnsembleKalmanProcesses.jl itself. `params` intentionally
# excludes ensemble size and max-iteration count (covered by the leaderboard charts) and
# instead lists the update-specific hyperparameters.
method_meta = {
"TEKI": {
"citation": "Chada et al., SIAM/ASA J. UQ, 2020",
"url": "https://doi.org/10.1137/17M114402X",
"source": "`opt_experiments/ensemble_kalman_processes` — EnsembleKalmanProcesses.jl `Inversion(prior)`.",
"math": (
"Ensemble Kalman Inversion (Schillings & Stuart, 2017, eqns. 4-5): each iteration moves every "
"ensemble member by the same linear, empirical-covariance-based correction toward the data,"
),
"math_eq": r"u_{n+1} = u_n + C^{ug}\left(C^{gg}+\Gamma\right)^{-1}\left(y - g(u_n)\right)",
"guardrails": (
"Tikhonov-style regularization is obtained by extending the state: the forward map is "
"augmented as `g ↦ [g; u]` and the data as `y ↦ [y; prior_mean]` with observation covariance "
"`blockdiag(Γ, prior_cov)`, so every step also pulls the ensemble back toward the prior "
"(`impose_prior=true`). A small `default_multiplicative_inflation` (1e-3) regularizes the "
"empirical covariance before it is inverted; `NoLocalization()` applies no covariance tapering; "
"`DefaultScheduler()` takes a fixed unit pseudo-time step each iteration."
),
"params": [
{"Parameter": "impose_prior", "Value": "true (Tikhonov-style state extension)"},
{"Parameter": "default_multiplicative_inflation", "Value": "1e-3"},
{"Parameter": "localization", "Value": "NoLocalization()"},
{"Parameter": "scheduler", "Value": "DefaultScheduler() — fixed unit step"},
],
"notes": "Runs until the whitened RMSE falls below the target or `N_iter` is exhausted; no reported pathology on this benchmark beyond the usual small-ensemble sampling error of stochastic EKI.",
},
"ETKI": {
"citation": "Schillings & Stuart, Numer. Math., 2017",
"url": "https://clima.github.io/EnsembleKalmanProcesses.jl/dev/",
"source": "`opt_experiments/ensemble_kalman_processes` — EnsembleKalmanProcesses.jl `TransformInversion(prior)`.",
"math": (
"Ensemble Transform Kalman Inversion applies the update in ensemble-anomaly space instead of "
"perturbing observations: normalized anomalies `X=(u-ū)/√(m-1)` and `Y=(g_ext-ḡ_ext)/√(m-1)` are "
"formed (with the same prior extension as TEKI), and a transform matrix built from `Y'Γ⁻¹Y` maps "
"the prior ensemble anomalies onto posterior anomalies, `u ← ū + X·T`."
),
"guardrails": (
"Same Tikhonov-style prior extension and `impose_prior=true` as TEKI; diagonal regularization "
"stabilizes the anomaly Gram matrix before it is inverted; `NoLocalization()`; the transform "
"update removes the perturbed-observation sampling noise present in stochastic EKI, which "
"generally lowers ensemble-collapse risk at small ensemble sizes."
),
"params": [
{"Parameter": "impose_prior", "Value": "true (Tikhonov-style state extension)"},
{"Parameter": "localization", "Value": "NoLocalization()"},
{"Parameter": "scheduler", "Value": "DefaultScheduler() — fixed unit step"},
],
"notes": "Same run-until-RMSE-target harness as TEKI; the transform (square-root) update trades stochastic sampling noise for a deterministic ensemble-anomaly map.",
},
"IEKF": {
"citation": "Iglesias, Inverse Problems, 2016",
"url": "https://doi.org/10.1088/0266-5611/32/2/025002",
"source": "`uq_experiments/GaussNewtonKalmanInversion` — EnsembleKalmanProcesses.jl `GaussNewtonInversion(prior)` (alias GNKI); Chada, Chen & Sanz-Alonso, 2021, Algorithm 4.1.",
"math": (
"Iterated Ensemble Kalman Filter with statistical linearization: each pseudo-time step `Δt` "
"relaxes the ensemble between its current position and a linearized Gauss-Newton solve of the "
"joint parameter/output covariance, evaluated against a fresh perturbed draw around the prior "
"mean rather than the ensemble's own mean — a Gauss-Newton normal-equation solve carried out "
"entirely in empirical-covariance space, without ever forming the model Jacobian explicitly."
),
"guardrails": (
"`perturbation_scale = 2/Δt - 1` rescales the prior-draw noise so the relaxation step's "
"stationary covariance equals the true posterior for every `Δt ∈ (0,1]`, not just `Δt=1`; "
"diagonal regularization + `NoLocalization()` stabilize the joint covariance solve; the pseudo-"
"step is fixed rather than adaptive (`DefaultScheduler(0.1)`, i.e. `Δt=0.1`)."
),
"params": [
{"Parameter": "pseudo-step Δt", "Value": "0.1 (DefaultScheduler(0.1))"},
{"Parameter": "perturbation_scale", "Value": "2/Δt - 1 = 19"},
{"Parameter": "localization", "Value": "NoLocalization()"},
],
"notes": "An iterative Gauss-Newton procedure, so — like other Newton-type ensemble methods — it can be sensitive to ensemble size at small N_ens; only the first `max_iter` (10 on L63, 15 on L96) of the `N_iter=20` allowed steps are scored.",
},
"UKI": {
"citation": "Huang, Huang & Stuart, Physica D, 2022",
"url": "https://clima.github.io/EnsembleKalmanProcesses.jl/dev/",
"source": "`opt_experiments/ensemble_kalman_processes` — EnsembleKalmanProcesses.jl `Unscented(prior; impose_prior=true)`.",
"math": (
"Unscented Kalman Inversion propagates `2·N_par + 1` deterministic sigma points (the symmetric "
"unscented transform) through the forward model instead of a random ensemble, forms their "
"weighted mean/covariance, and performs the same Kalman-style analysis step as TEKI on this "
"reduced sigma-point set."
),
"guardrails": (
"`α_reg ∈ (0,1]` shrinks the analysis mean toward the prior mean each step, guarding against "
"covariance collapse; `impose_prior=true` adds the same TEKI-style `y_ext`/`Γ_ext` extension; "
"the symmetric sigma-point scheme (`2·N_par+1` points) is used by default rather than the "
"cheaper simplex scheme."
),
"params": [
{"Parameter": "α_reg", "Value": "1.0 (regularization toward prior mean disabled by default)"},
{"Parameter": "sigma_points", "Value": '"symmetric" (2·N_par+1 points)'},
{"Parameter": "impose_prior", "Value": "true"},
],
"notes": "Sigma-point count is fixed by the parameter dimension, not swept as an ensemble size — UKI's per-iteration cost is dimension-driven rather than N_ens-driven.",
},
"ABC": {
"citation": "Csilléry et al., Trends in Ecology & Evolution, 2010",
"url": "https://doi.org/10.1016/j.tree.2010.04.001",
"source": "`uq_experiments/approximate_bayesian_computation` — simulation-based rejection sampling (no EKP, no surrogate).",
"math": (
"Each round draws `N_ens` i.i.d. prior candidates `θ ~ π(θ)`, evaluates the true forward model "
"`G(θ)`, and accepts a candidate iff its whitened misfit falls inside a chi-squared acceptance "
"region; accepted candidates across all rounds approximate draws from the ABC posterior."
),
"math_eq": r"\left(G(\theta)-y\right)^{\top}(2R)^{-1}\left(G(\theta)-y\right) \le \chi^2_{n_y}(1-\alpha_{\text{reject}})",
"guardrails": (
"The acceptance threshold is set exactly to the `(1-alpha_reject)` quantile of `χ²(n_y)` rather "
"than hand-tuned; the final pushforward resamples `n_pushforward_samples=1000` draws from a "
"Gaussian fit to the accepted pool, smoothing a possibly sparse accepted set into a usable "
"posterior approximation; batch sizes are deliberately matched to IEKF's `N_ens_sizes`/`max_iter` "
"so the N_ens×k budget axis is directly comparable across methods on the leaderboard."
),
"params": [
{"Parameter": "alpha_reject", "Value": "0.05"},
{"Parameter": "n_pushforward_samples", "Value": "1000"},
],
"notes": "Acceptance rate collapses quickly in higher dimensions (the same \"curse of dimensionality\" documented for HM below); there is no adaptive tightening of the acceptance region as rounds proceed, so all cost-efficiency comes from the batch-size choice. The Gaussian fit to the accepted pool additionally requires at least `p+2` accepted posterior samples before it can be estimated (`p` = parameter dimension); in higher-dimensional experiments this minimum is never reached within the 10,000-evaluation cap, so the pushforward Gaussian is never fit.",
},
"HM": {
"citation": "Williamson et al. 2013; King et al. 2025",
"url": "https://example.com/hm",
"source": "`uq_experiments/HistoryMatching` — custom (non-EKP) wave/rejection scheme.",
"math": (
"History Matching runs a series of waves. At wave k, an independent GP emulator is fit per "
"whitened output mode on an accumulated training set: the current ensemble plus all training "
"points from historical waves that lie within the current NROY space (points that have been "
"ruled out are dropped, so the emulator concentrates its accuracy on the shrinking NROY region); "
"a candidate survives the wave iff "
"its implausibility (whitened, GP-predicted misfit) is below a chi-squared threshold, and the "
"surviving (\"Not Ruled Out Yet\", NROY) region is intersected across all waves run so far."
),
"math_eq": r"I^2(\theta) = \sum_j \frac{\left(\mu_j(\theta)-y_j\right)^2}{1+\sigma_j^2(\theta)} < \chi^2_{k_R}(0.95)",
"guardrails": (
"GP emulators use an automatic-relevance-determination squared-exponential kernel (`SEArd`), "
"refit every wave via `GaussianProcesses.optimize!`; NROY testing exploits monotonicity — a "
"candidate ruled out by an earlier wave is never re-tested by later waves — cutting GP-predict "
"cost from O(waves×batch) to O(surviving candidates); a Poisson/Garwood upper-confidence-bound "
"rule (`poisson_rate_upper_bound`, `early_stop_alpha=1e-6`) aborts a rejection-sampling sweep "
"early once even its best-case acceptance rate can no longer plausibly reach the wanted sample "
"count within the remaining budget, rather than burning the full `max_samples` budget on a "
"doomed sweep; inputs/outputs are PCA-whitened (`retain_var_input`/`retain_var`) before fitting."
),
"params": [
{"Parameter": "confidence (implausibility threshold)", "Value": "0.95 → quantile(Chisq(k_R), 0.95)"},
{"Parameter": "n_candidate_batch", "Value": "2000 per wave"},
{"Parameter": "retain_var (output PCA)", "Value": "0.99"},
{"Parameter": "retain_var_input (input PCA)", "Value": "0.9"},
{"Parameter": "early_stop_alpha", "Value": "1e-6"},
],
"notes": "Acceptance rate collapses toward zero after a couple of waves on higher-dimensional problems (the repo's own documented \"curse of dimensionality\"); the early-stop rule only changes how quickly a doomed sweep gives up, not the eventual pass/fail outcome.",
},
"CES-EKI-DMC": {
"citation": "Cleary et al., J. Comput. Phys., 2021",
"url": "https://doi.org/10.1016/j.jcp.2020.109716",
"source": "`uq_experiments/calibrate_emulate_sample` — Calibrate-Emulate-Sample, calibrate stage: `Inversion()` + `DataMisfitController`.",
"math": (
"Three-stage Calibrate-Emulate-Sample. Calibrate: run EKI to generate an ensemble of "
"`(θ, G(θ))` training pairs. Emulate: fit a random-feature surrogate `Ĝ(θ)` on those pairs. "
"Sample: draw from `π(θ|y) ∝ exp(-½‖y-Ĝ(θ)‖²_Γ)·π(θ)` against the cheap surrogate using "
"preconditioned Crank-Nicolson Metropolis-Hastings (pCN-MH), rather than the true forward model."
),
"guardrails": (
"Calibrate stage uses an adaptive `DataMisfitController(terminate_at=2.0)` schedule so the EKI "
"pseudo-time step grows automatically and training points concentrate near the posterior, "
"rather than requiring a hand-tuned fixed schedule (`terminate_at=2.0` approximates the "
"finite-time posterior); emulate stage keeps 99% of variance in a likelihood-informed PCA "
"encoding before fitting the `ScalarRandomFeatureInterface`; sample stage auto-tunes the pCN "
"step size (`optimize_stepsize`, 2,000-sample dry run from `init_stepsize=0.2`) before the full "
"100,000-sample chain with a 2,000-sample burn-in."
),
"params": [
{"Parameter": "scheduler (calibrate)", "Value": "DataMisfitController(terminate_at=2.0)"},
{"Parameter": "retain_var (emulate PCA)", "Value": "0.99"},
{"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"},
{"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"},
{"Parameter": "init_stepsize", "Value": "0.2 (auto-tuned before the full chain)"},
],
"notes": "The emulator is fit only once, after calibration finishes (\"after-optimize\"), so sampled-posterior quality depends entirely on how well the calibrate-stage ensemble has already covered the posterior region.",
},
"CES-EKI-CONST": {
"citation": "Cleary et al., J. Comput. Phys., 2021",
"url": "https://doi.org/10.1016/j.jcp.2020.109716",
"source": "`uq_experiments/calibrate_emulate_sample` — Calibrate-Emulate-Sample, calibrate stage: `Inversion()` + fixed-step scheduler.",
"math": "Identical Calibrate-Emulate-Sample pipeline to CES-EKI-DMC (see that entry for the emulate/sample math).",
"guardrails": (
"Calibrate stage uses a fixed-step `DefaultScheduler(0.1)` instead of the adaptive "
"`DataMisfitController`, trading adaptivity for a simpler, more predictable schedule; emulate "
"and sample stages are unchanged from CES-EKI-DMC (same PCA retention, feature counts, and "
"MCMC step-size tuning)."
),
"params": [
{"Parameter": "scheduler (calibrate)", "Value": "DefaultScheduler(0.1) — fixed step"},
{"Parameter": "retain_var (emulate PCA)", "Value": "0.99"},
{"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"},
{"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"},
],
"notes": "Same after-optimize emulator caveat as CES-EKI-DMC; the fixed step trades the DMC variant's adaptive posterior-concentration for predictability.",
},
"CES-IEKF-CONST": {
"citation": "Cleary et al., J. Comput. Phys., 2021; Iglesias, Inverse Problems, 2016",
"url": "https://doi.org/10.1016/j.jcp.2020.109716",
"source": "`uq_experiments/calibrate_emulate_sample` — Calibrate-Emulate-Sample, calibrate stage: `GaussNewtonInversion(prior)` + fixed-step scheduler.",
"math": "Identical Calibrate-Emulate-Sample pipeline to CES-EKI-DMC, but the calibrate stage runs the IEKF/GNKI Gauss-Newton update (see the IEKF entry) instead of plain EKI.",
"guardrails": (
"Calibrate stage uses `GaussNewtonInversion(prior)` with a fixed pseudo-step `DefaultScheduler(0.1)`; "
"emulate and sample stages are unchanged from CES-EKI-DMC (same PCA retention, feature counts, "
"and MCMC step-size tuning)."
),
"params": [
{"Parameter": "scheduler (calibrate)", "Value": "DefaultScheduler(0.1) — fixed pseudo-step"},
{"Parameter": "retain_var (emulate PCA)", "Value": "0.99"},
{"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"},
{"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"},
],
"notes": "Same after-optimize emulator caveat as CES-EKI-DMC, with the Gauss-Newton calibrate step's usual small-ensemble sensitivity (see IEKF).",
},
"CES-EKS": {
"citation": "Cleary et al., J. Comput. Phys., 2021; Garbuno-Inigo, Nüsken and Reich, SIAM J. Appl. Dyn. Syst., 2020",
"url": "https://doi.org/10.1016/j.jcp.2020.109716",
"source": "`uq_experiments/calibrate_emulate_sample` — `CALIBRATE_METHOD=eks` (calibrate stage `Sampler(prior)` + `EKSStableScheduler()`), `DIM_REDUCTION=PCA` (leaderboard key `ces-eks-pca`).",
"math": (
"Calibrate-Emulate-Sample whose calibrate stage runs the Ensemble Kalman Sampler (see the EKS entry) instead of "
"plain EKI. Calibrate: run the sampler to generate `(θ, G(θ))` training pairs; iteration `k` trains on "
"the pairs from the first `k` steps. Emulate: fit a scalar random-feature surrogate `Ĝ(θ)` on those "
"pairs. Sample: draw from `π(θ|y) ∝ exp(-½‖y-Ĝ(θ)‖²_Γ)·π(θ)` against the cheap surrogate using "
"preconditioned Crank-Nicolson Metropolis-Hastings (pCN-MH), rather than the true forward model."
),
"guardrails": (
"Calibrate stage: `Sampler(prior)` with the data-dependent `EKSStableScheduler()`. The random-feature "
"kernel has a `1e-6` diagonal nugget. The nullspace-noise injector is switched off for this variant, so directions dropped by the "
"input PCA are decoded at the prior mean rather than filled with prior noise (the experiment config "
"notes that injecting noise inflated the posterior on L96 flux-force, with output spread about 2x too "
"wide and no contraction with `k`). Sample stage auto-tunes the pCN step size (`optimize_stepsize`, "
"2,000-sample dry run) before the full chain."
),
"params": [
{"Parameter": "calibrate sampler / scheduler", "Value": "Sampler(prior) / EKSStableScheduler()"},
{"Parameter": "n_features / n_features_opt", "Value": "100/60 (L63), 200/160 (L96 variants)"},
{"Parameter": "kernel nugget", "Value": "1e-6"},
{"Parameter": "MCMC samples / burn-in", "Value": "100,000 / 2,000 (pCN-MH)"},
{"Parameter": "init_stepsize", "Value": "0.2 (L63) / 0.01 (L96), auto-tuned before the full chain"},
{"Parameter": "nullspace noise injector", "Value": "off (threshold Inf)"},
],
"notes": (
"The emulator is fit once per iteration `k` on the sampler's first `k` steps, so quality depends on "
"how well the calibrate-stage ensemble covers the posterior region. In the current UQ data "
"(c = 1.5, best reliable ensemble size) the mean budget is 5.3 on L63, 6.0 on L96 const-force, 87.5 "
"on vec-force (0% failures) and 207 on NN flux-force, which fails on at least 5% of seeds at every "
"size (15% at the best one)."
),
},
"ADAM": {
"citation": "Kingma & Ba, ICLR, 2015",
"url": "https://doi.org/10.48550/arXiv.1412.6980",
"source": "`opt_experiments/adam` — custom (non-Flux) Adam implementation.",
"math": (
"Adam gradient descent on the whitened least-squares loss `L(θ)=½‖R^{-1/2}(y-G(θ))‖²`: "
"first/second raw-gradient moments `m_t=β₁m_{t-1}+(1-β₁)g_t`, `v_t=β₂v_{t-1}+(1-β₂)g_t²` are "
"bias-corrected to `m̂_t, v̂_t` and applied as `θ_{t+1}=θ_t - α·m̂_t/(√v̂_t+ε)`. The gradient "
"`g_t` comes from a whitened Jacobian obtained via `ForwardDiff.jl` forward-mode automatic "
"differentiation through the Lorenz solver itself — not a surrogate."
),
"guardrails": (
"`N_ens` initial-condition perturbations are drawn and averaged into the gradient at every step "
"to reduce gradient noise from the chaotic forward model (`N_ens=1` here, i.e. a single-sample "
"stochastic gradient); beyond Adam's own adaptive per-parameter step scaling there is no line "
"search, trust region, or restart mechanism."
),
"params": [
{"Parameter": "α (step size)", "Value": "0.001"},
{"Parameter": "β₁, β₂", "Value": "0.9, 0.999"},
{"Parameter": "ε", "Value": "1e-8"},
{"Parameter": "N_ens (IC perturbations)", "Value": "1"},
],
"notes": "**Empirically gets stuck in local minima close to the initial parameter draw, regardless of step size** — observed running this benchmark. Consistent with Adam having no mechanism (trust region, restart, ensemble diversity) to escape a bad basin once its gradient estimate vanishes there.",
},
"LM": {
"citation": "Levenberg, 1944; Marquardt, 1963; Fletcher, 1971",
"url": "https://doi.org/10.1090/qam/10666",
"source": "`opt_experiments/levenberg_marquardt` — custom Levenberg-Marquardt implementation.",
"math": (
"Damped Gauss-Newton: each step solves the damped normal equations for the parameter update "
"`Δθ`, using a whitened Jacobian/residual `(J̃, r̃)` obtained via `ForwardDiff.jl`, then accepts "
"the step only if the reduction ratio `ρ` (actual vs. predicted misfit decrease) is favorable."
),
"math_eq": r"\left(\tilde J^{\top}\tilde J + \lambda\,\mathrm{diag}(d)\right)\Delta\theta = \tilde J^{\top}\tilde r",
"guardrails": (
"Damping `λ` adapts every step: ×4 (capped at `λ_max`) on a rejected or non-finite step "
"(`ρ<0.25` or NaN), ÷3 (floored at `λ_min`) on an accepted step (`ρ>0.75`), interpolating between "
"gradient descent (large λ) and Gauss-Newton (small λ); a non-finite `Δθ` is handled gracefully "
"(`ρ=NaN → !isfinite(ρ) → λ↑`) rather than crashing the run."
),
"params": [
{"Parameter": "λ₀ (initial damping)", "Value": "1.0"},
{"Parameter": "λ growth / shrink factor", "Value": "×4 (reject) / ÷3 (accept)"},
{"Parameter": "λ bounds", "Value": "[1e-10, 1e8]"},
{"Parameter": "N_ens (IC perturbations)", "Value": "1"},
],
"notes": "**Empirically also gets stuck in local minima near the initial condition, regardless of the damping schedule** — the same failure mode as ADAM, unsurprising for any single-trajectory local optimizer on this multimodal, chaotic-dynamics calibration surface.",
},
"GBOED-TMCMC": {
"citation": "Holthuijzen et al. 2026",
"url": "https://arxiv.org/abs/2508.13071",
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — Goal-oriented BOED, Algorithm 2 of Holthuijzen et al. 2026.",
"math": (
"Each iteration: (i) fits a GP surrogate per output dimension to all `(θ, G(θ))` collected so "
"far, (ii) draws posterior samples from the current surrogate-based posterior via Transitional/"
"Sequential MCMC (Ching & Chen, 2007), (iii) selects the next batch of forward-model evaluations "
"by maximizing a closed-form Expected Information Gain (EIG) criterion targeted at the quantity "
"of interest, using the GP's predictive covariance."
),
"math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}",
"guardrails": (
"Candidate batches are optimized with `Optim.jl`'s `Fminbox(LBFGS())` under box constraints, "
"with a hard cap on raw EIG evaluations (`eig_call_limit`) and an outer-iteration cap "
"(`eig_outer_iters`) enforced manually — since Optim's own call limits don't cumulate across "
"Fminbox's outer barrier rounds — logging a warning that the returned point may be "
"\"under-optimized\" if either cap is hit, rather than failing silently; a diagonal jitter "
"(`eig_jitter`, relative to the GP signal variance) stabilizes the covariance solves/log-"
"determinants; batch points can be chosen greedily, jointly, or via a hybrid strategy that "
"switches from greedy to joint optimization once a greedy step's cost exceeds a threshold."
),
"params": [
{"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"},
{"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"},
{"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"},
{"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"},
{"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"},
{"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
{"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"},
{"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
{"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
],
"notes": "**Both the Transitional MCMC (TMCMC) posterior sampling and the EIG batch optimization contribute to performance.** TMCMC needs a sufficiently large population (`n_posterior_samples`) to represent the posterior, particularly in higher dimensions; its cost grows roughly linearly with population size, and `tmcmc_burnin`/`tmcmc_thin` multiply the per-stage cost. The EIG objective uses only a small posterior subsample (`n_eig_posterior_samples`), as EIG costs ~n³ per output mode. **The inner EIG-maximization loop is a frequent source of problems (hitting the call-limit / outer-iteration guardrails, i.e. \"under-optimized\" batches), and every iteration slows down as the cumulative GP training set grows**, which is why the higher-dimensional L96 cases use a smaller `max_iters`.",
},
"LM-KGMM": {
"citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025",
"url": "https://arxiv.org/abs/2503.18054",
"source": "`opt_experiments/score_based_lm` — Levenberg-Marquardt with a GFDT/KGMM Jacobian surrogate.",
"math": (
"Same LM step as above, but the Jacobian is not differentiated through the solver — it is "
"estimated from trajectory data via the generalized fluctuation-dissipation theorem (GFDT): the "
"response of a fitted statistic `A(x)` to a parameter perturbation is approximated by a lag-"
"integrated, Tukey-tapered correlation with a \"conjugate\" variable `B(x)` built from the score "
"function `s(x)=∇log p(x)`, then chained through `dφ/dm` to the summary-statistic Jacobian."
),
"math_eq": r"\frac{d\,\mathbb{E}[A]}{d\theta} \approx \int_0^{\tau_{\max}} \left\langle A(x_{t+\tau})\,B(x_t)\right\rangle\, w(\tau)\, d\tau",
"guardrails": (
"The score `s(x)` is estimated with a k-means Gaussian-mixture score estimator (KGMM): the "
"trajectory is clustered into components whose mixture has a closed-form score. "
"`stein_recalibrate` rescales the estimated score so `-E[x·s(x)ᵀ]=I` holds on the calibration "
"ensemble (a built-in consistency check/correction); `kgmm_ridge` shrinks per-cluster "
"covariances toward the global covariance to avoid singular clusters; `is_collapsed_window` "
"(minimum trajectory variance below a tolerance) detects windows that have collapsed onto a "
"fixed point — GFDT is undefined there, since there is no invariant measure to differentiate — "
"and falls back to a direct `ForwardDiff.jl` Jacobian on those draws; total cost is capped at "
"`outer_iter × N_ens ≤ budget_total` per cell, trading ensemble size against iteration count "
"under a fixed evaluation budget."
),
"params": [
{"Parameter": "kgmm_ridge", "Value": "0.1"},
{"Parameter": "tukey_alpha (response taper)", "Value": "0.75"},
{"Parameter": "budget_total (outer_iter × N_ens)", "Value": "500"},
{"Parameter": "collapse tolerance (min variance)", "Value": "1e-2"},
],
"notes": "**Much quicker to apply than LM-DSM** — fitting a k-means/Gaussian-mixture score is far cheaper than training a neural score network, so LM-KGMM stayed tractable on problems where LM-DSM's training cost made it impractical.",
},
"LM-DSM": {
"citation": "Giorgini, Bischoff & Souza, arXiv:2509.19660, 2025",
"url": "https://arxiv.org/abs/2509.19660",
"source": "`opt_experiments/score_based_lm` — Levenberg-Marquardt with a GFDT/DSM Jacobian surrogate.",
"math": "Identical LM + GFDT scheme as LM-KGMM (see that entry's update equation), but the score `s(x)=∇log p(x)` is estimated by Denoising Score Matching (DSM) instead of KGMM.",
"guardrails": (
"A neural network is trained to denoise samples perturbed by Gaussian noise at a single fixed "
"scale `σ`; its optimal denoiser equals the noised density's score (Vincent, 2011), and as "
"`σ→0` this recovers the true score at the cost of a harder training problem. `σ` is fixed per "
"benchmark rather than annealed, chosen empirically as the smallest noise level at which the "
"network still receives a usable learning signal. Same Stein recalibration and collapsed-window "
"`ForwardDiff.jl` fallback as LM-KGMM; same Tukey-tapered response integral and evaluation-"
"budget cap. **Score model & training:** a small network (MLP for L63, periodic 1-D U-Net for "
"L96) is trained with Adam on the current statistics window each outer iteration — a full "
"initial fit, then short warm-started refits that carry the weights across iterations; on the "
"larger L96 vec/flux cases only every 20th window sample is used for training (the GFDT score "
"is still evaluated on the full window)."
),
"params": [
{"Parameter": "Score network", "Value": "3×128 swish MLP (L63) / circular 1-D U-Net, ~66k params (L96 variants)"},
{"Parameter": "DSM training epochs (initial → warm refit)", "Value": "500 → 50 (L63) / 150 → 20 (L96 const) / 150 → 10 (L96 vec, flux)"},
{"Parameter": "DSM optimizer", "Value": "Adam, lr 1e-3 / 8e-4 initial → 3e-4 warm; batch 256 (L63) / 128 (L96); 10% validation hold-out"},
{"Parameter": "σ (DSM noise scale)", "Value": "0.2 (L63) / 0.025 (L96 variants)"},
{"Parameter": "tukey_alpha (response taper)", "Value": "0.75"},
{"Parameter": "budget_total (outer_iter × N_ens)", "Value": "500"},
{"Parameter": "collapse tolerance (min variance)", "Value": "1e-2"},
],
"notes": "**Training the DSM score network is very slow, even on moderately sized problems** — a 5-iteration smoke test on the higher-dimensional L96 forcing variants took ~39 minutes, extrapolating to several hours for a full run, and higher-dimensional experiments were killed after ~24h without finishing a sweep. Wall-clock cost, not accuracy, was the binding constraint on scaling LM-DSM to larger problems.",
},
"ADAM-KGMM": {
"citation": "Giorgini, Bischoff & Souza, arXiv:2503.18054, 2025",
"url": "https://arxiv.org/abs/2503.18054",
"source": "`opt_experiments/score_based_adam` — Adam with a GFDT/KGMM gradient surrogate.",
"math": (
"Same Adam moment-based update as above, but the gradient `g_t` is not differentiated through "
"the solver — it is estimated from trajectory data via the generalized fluctuation-dissipation "
"theorem (GFDT), the same lag-integrated, Tukey-tapered response construction used by LM-KGMM "
"(see that entry's update equation), chained through `dφ/dm` to the whitened-loss gradient "
"instead of a Gauss-Newton Jacobian."
),
"guardrails": (
"Same GFDT machinery as LM-KGMM: the score `s(x)=∇log p(x)` is estimated with a k-means "
"Gaussian-mixture score estimator (KGMM), `stein_recalibrate` rescales it so `-E[x·s(x)ᵀ]=I` "
"holds on the calibration ensemble, `kgmm_ridge` shrinks per-cluster covariances toward the "
"global covariance, and `is_collapsed_window` falls back to a direct `ForwardDiff.jl` gradient "
"on windows that have collapsed onto a fixed point; total cost is capped at "
"`outer_iter × N_ens ≤ budget_total` per cell (5,000 here, vs 500 for the LM variants), "
"trading ensemble size against iteration count under a fixed evaluation budget; beyond that, Adam's own adaptive per-parameter "
"step scaling is the only guardrail on the step itself. Unlike `β₁`/`β₂`/`ε` (shared across all "
"cases), the step size `α` is set per-experiment from the case's prior — sized to ~0.25× the "
"mean prior standard deviation, so a run of consistent-sign steps crosses ~1 prior std in ~4 "
"outer iterations; a single global value tuned for one benchmark's residual scaling left Adam "
"unable to traverse even the L63/L96-const prior within its iteration budget."
),
"params": [
{"Parameter": "α (step size)", "Value": "Per-case, ~0.25× prior std (0.025–0.75 across benchmarks)"},
{"Parameter": "β₁, β₂", "Value": "0.9, 0.999"},
{"Parameter": "kgmm_ridge", "Value": "0.1"},
{"Parameter": "tukey_alpha (response taper)", "Value": "0.75"},
{"Parameter": "budget_total (outer_iter × N_ens)", "Value": "5000"},
{"Parameter": "collapse tolerance (min variance)", "Value": "1e-2"},
],
"notes": "**Inherits Adam's local-minimum susceptibility** (no trust region, restart, or ensemble diversity to escape a bad basin) **and LM-KGMM's cheaper-than-DSM score estimation** — the k-means/Gaussian-mixture score fit is far less costly than training a neural score network, so ADAM-KGMM stayed tractable on problems where a DSM-based Adam variant would not.",
},
"ADAM-DSM": {
"citation": "Giorgini, Bischoff & Souza, arXiv:2509.19660, 2025",
"url": "https://arxiv.org/abs/2509.19660",
"source": "`opt_experiments/score_based_adam` — Adam with a GFDT/DSM gradient surrogate.",
"math": "Identical Adam + GFDT scheme as ADAM-KGMM (see that entry's update equation), but the score `s(x)=∇log p(x)` is estimated by Denoising Score Matching (DSM) instead of KGMM.",
"guardrails": (
"Same DSM training scheme as LM-DSM: a neural network is trained to denoise samples perturbed "
"by Gaussian noise at a single fixed scale `σ`, whose optimal denoiser equals the noised "
"density's score (Vincent, 2011); `σ` is fixed per benchmark rather than annealed, chosen "
"empirically as the smallest noise level at which the network still receives a usable learning "
"signal. Same Stein recalibration and collapsed-window `ForwardDiff.jl` fallback, and the same "
"Tukey-tapered response integral and evaluation-budget cap (5,000 evaluations) as ADAM-KGMM. "
"Same per-case "
"step size `α` (shared `β₁`/`β₂`/`ε`) as ADAM-KGMM — sized to ~0.25× the case's prior standard "
"deviation, since a single global value tuned for one benchmark's residual scaling left Adam "
"unable to traverse even the L63/L96-const prior within its iteration budget. **Score model & "
"training:** a small network (MLP for L63, periodic 1-D U-Net for L96) is trained with Adam on "
"the current statistics window each outer iteration — a full initial fit, then short "
"warm-started refits that carry the weights across iterations; on the larger L96 vec/flux "
"cases only every 20th window sample is used for training (the GFDT score is still evaluated "
"on the full window)."
),
"params": [
{"Parameter": "α (step size)", "Value": "Per-case, ~0.25× prior std (0.025–0.75 across benchmarks)"},
{"Parameter": "β₁, β₂", "Value": "0.9, 0.999"},
{"Parameter": "Score network", "Value": "3×128 swish MLP (L63) / circular 1-D U-Net, ~66k params (L96 variants)"},
{"Parameter": "DSM training epochs (initial → warm refit)", "Value": "500 → 50 (L63) / 150 → 20 (L96 const) / 150 → 10 (L96 vec, flux)"},
{"Parameter": "DSM optimizer", "Value": "Adam, lr 1e-3 / 8e-4 initial → 3e-4 warm; batch 256 (L63) / 128 (L96); 10% validation hold-out"},
{"Parameter": "σ (DSM noise scale)", "Value": "0.2 (L63) / 0.025 (L96 variants)"},
{"Parameter": "tukey_alpha (response taper)", "Value": "0.75"},
{"Parameter": "budget_total (outer_iter × N_ens)", "Value": "5000"},
{"Parameter": "collapse tolerance (min variance)", "Value": "1e-2"},
],
"notes": "**Inherits both LM-DSM's slow DSM-training cost and Adam's local-minimum susceptibility** — training the DSM score network is the binding wall-clock cost (as with LM-DSM), and Adam's lack of a trust region or ensemble diversity leaves it as prone to getting stuck in a bad basin as ADAM-KGMM.",
},
"RL-DDPG-SHORT": {
"citation": "Lillicrap et al., ICLR, 2016",
"url": "https://arxiv.org/abs/1509.02971",
"source": "`opt_experiments/ReinforcementLearning` — DDPG actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `UPDATE_FREQUENCY=short`.",
"math": (
"The unknown forcing parameterization is the **policy** and the whitened statistics misfit "
"`Φ_k=‖y-G(x_{T_k})‖²_Γ` defines the **reward** `r_k=Φ_{k-1}-Φ_k`. A small critic MLP `q(x,F)` is "
"trained on replay-buffer Bellman targets `-(1-γ)Φ_k+γ·q(x_{k+1},π(x_{k+1}))` (the `Φ_{k-1}` "
"offset cancels exactly), and the policy parameters are ascended along `∇_F q` averaged over "
"buffer states with Adam. No Jacobian of the simulator is ever formed — the gradient comes "
"from differentiating the learned critic. **The prior enters in three ways, but not as a "
"penalty:** the initial policy is a random draw from the prior, the policy is stored in "
"prior-whitened coordinates `F=μ+Lθ̃` (`L` = Cholesky factor of the prior covariance) so the "
"Adam step and the Gaussian exploration noise are measured in prior standard deviations "
"(following the prior's correlations), and the critic sees the whitened action. The reward "
"is the misfit only. In the *short* variant one RL step advances a "
"sliding statistics window by 1/10 of its length, so there are ~10 RL updates per window of "
"simulated time, each costing a fraction of a forward evaluation."
),
"math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q_\vartheta(x,F)\right]_{F=F_\theta}\right)",
"guardrails": (
"`γ=0.5<1` is essential: for a state-independent policy at `γ=1` the reward telescopes and "
"`∇_F Q` vanishes, so `γ=1` reproduces the no-learning control. A Polyak-averaged target critic "
"(`τ=0.05`) stabilizes the bootstrap; critic inputs/targets are standardized from warm-up data "
"and frozen; Gaussian exploration noise (0.1 prior std) is added to the applied action, with a "
"warm-up phase before the first update. The statistic is computed over a trailing window of "
"the full length `W` (not the short step) so it is not biased by a shortened estimation "
"window. Cost is counted as simulation only (1 initial full evaluation + stride/T per step); "
"per-cell budget is capped at 10000 units."
),
"params": [
{"Parameter": "γ (discount)", "Value": "0.5"},
{"Parameter": "Polyak rate", "Value": "0.05"},
{"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
{"Parameter": "batch size / updates per step", "Value": "64 / 1"},
{"Parameter": "steps per window / warm-up steps / buffer size", "Value": "10 / 50 / 5000"},
{"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
{"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
],
"notes": (
"**Cost advantage is partly structural** — the continuing RL trajectory pays spin-up once rather "
"than every step (for L63 `T_start/T=0.75`), and the convergence test runs at every stride on "
"overlapping windows, so scores are not directly comparable to methods that re-pay spin-up each "
"evaluation. **Not tuned per case.** Converges on every seed on "
"L63 and L96-const, but the higher-dimensional forcings are the hard case for a learned "
"critic: on L96 flux-force and vec-force it reaches the targets on only a few of 30 seeds "
"(flux: 1 at target 1.1, 5 at 1.2; vec: 2 at 1.1, 4 at 1.2) and on none at 1.0 within the budget."
),
},
"RL-DDPG": {
"citation": "Lillicrap et al., ICLR, 2016",
"url": "https://arxiv.org/abs/1509.02971",
"source": "`opt_experiments/ReinforcementLearning` — DDPG actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `UPDATE_FREQUENCY=long` (leaderboard name `rl-ddpg-long`).",
"math": (
"Same DDPG actor-critic scheme as RL-DDPG-SHORT (see that entry's update equation): the unknown "
"forcing parameterization is the **policy**, the decrease in the whitened statistics misfit "
"`Φ_k` is the **reward**, and the policy parameters are ascended along `∇_F q` of a small learned "
"critic. The prior enters as in RL-DDPG-SHORT: a random prior draw as the initial policy, "
"prior-whitened policy coordinates (step and exploration noise in prior standard deviations), "
"and no prior penalty in the reward. Here one RL step is the whole statistics window — exactly one standard forward "
"evaluation (spin-up + window), the same statistical window as every other method on the "
"leaderboard — so there is one RL update per forward evaluation and each step costs 1 unit."
),
"math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q_\vartheta(x,F)\right]_{F=F_\theta}\right)",
"guardrails": (
"Same `γ=0.5<1` (at `γ=1` the critic has no `F`-dependence for a state-independent policy), "
"Polyak target critic (`τ=0.05`), frozen input/target standardization, and Gaussian exploration "
"noise (0.1 prior std) as RL-DDPG-SHORT. The statistic `G_k` is exactly the standard forward map "
"used by every other method, and the spin-up is paid on every step; per-cell budget is capped "
"at 10000 units."
),
"params": [
{"Parameter": "γ (discount)", "Value": "0.5"},
{"Parameter": "Polyak rate", "Value": "0.05"},
{"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
{"Parameter": "batch size / updates per step", "Value": "64 / 1"},
{"Parameter": "steps per window / warm-up steps / buffer size", "Value": "1 / 20 / 2000"},
{"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
{"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
],
"notes": (
"**One RL update per forward evaluation, so far more expensive than RL-DDPG-SHORT** — the short "
"variant makes ~10× more updates per unit of cost and pays spin-up only once. **Not tuned per "
"case.** Converges reliably on L63 and L96-const, but never "
"reached the targets on L96 flux-force or vec-force within the budget."
),
},
"RL-TD3-SHORT": {
"citation": "Fujimoto, van Hoof & Meger, ICML, 2018",
"url": "https://arxiv.org/abs/1802.09477",
"source": "`opt_experiments/ReinforcementLearning` — TD3 actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `ALGORITHM=td3`, `UPDATE_FREQUENCY=short`.",
"math": (
"Same actor-critic formulation as RL-DDPG-SHORT (see that entry): the unknown forcing "
"parameterization is the **policy**, the decrease in the whitened statistics misfit is the "
"**reward**, and the policy parameters are ascended along `∇_F q` of a learned critic. TD3 "
"replaces the single critic with **twin critics** whose minimum forms the Bellman target "
"`-(1-γ)Φ_k+γ·min_i q_i(x_{k+1},π(x_{k+1}))`, and the actor ascends critic 1 only. The prior "
"enters as in RL-DDPG-SHORT (random prior draw as the initial policy; prior-whitened policy "
"coordinates for the Adam step, exploration and target-smoothing noise; no prior penalty in "
"the reward). In the *short* variant one RL step advances a sliding statistics window by 1/10 of its length, so "
"there are ~10 RL updates per window of simulated time, each costing a fraction of a forward "
"evaluation."
),
"math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q^{(1)}_\vartheta(x,F)\right]_{F=F_\theta}\right)",
"guardrails": (
"The three TD3 robustness heuristics are switched on relative to RL-DDPG: **twin critics** with a "
"min-target (reduces critic overestimation), **delayed policy updates** (the actor and the "
"Polyak target critics update once per 2 critic updates), and **target-policy smoothing** "
"(clipped Gaussian noise added to the target action, in prior-std units, since the actor is a "
"single vector). As for RL-DDPG there is no target actor. Same `γ=0.5<1`, Polyak rate, frozen "
"input/target standardization, 0.1 prior-std Gaussian exploration noise and trailing window of "
"full length `W` as RL-DDPG-SHORT; cost counts simulation only (1 initial evaluation + stride/T "
"per step), capped at 10000 units per cell."
),
"params": [
{"Parameter": "γ (discount)", "Value": "0.5"},
{"Parameter": "Polyak rate", "Value": "0.05"},
{"Parameter": "critics / policy delay", "Value": "2 / 2"},
{"Parameter": "target smoothing noise (clip)", "Value": "0.2 (0.5), prior-std units"},
{"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
{"Parameter": "batch size / updates per step", "Value": "64 / 1"},
{"Parameter": "steps per window / warm-up steps / buffer size", "Value": "10 / 50 / 5000"},
{"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
{"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
],
"notes": (
"**Same structural cost advantage as RL-DDPG-SHORT** — the continuing trajectory pays spin-up "
"once and the convergence test runs on overlapping windows, so scores are not directly "
"comparable to methods that re-pay spin-up every evaluation. **Not tuned per case.** "
"Converges on every seed on L63 and L96-const (slower than RL-DDPG-SHORT on L63 at targets "
"1.0–1.1, similar at 1.2); on the higher-dimensional forcings it is weaker than RL-DDPG-SHORT: "
"it never reaches the targets on L96 flux-force, and on vec-force only 1 of 30 seeds does "
"(target 1.2) within the budget."
),
},
"RL-TD3": {
"citation": "Fujimoto, van Hoof & Meger, ICML, 2018",
"url": "https://arxiv.org/abs/1802.09477",
"source": "`opt_experiments/ReinforcementLearning` — TD3 actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `ALGORITHM=td3`, `UPDATE_FREQUENCY=long` (leaderboard name `rl-td3-long`).",
"math": (
"Same TD3 actor-critic scheme as RL-TD3-SHORT (twin critics with a min-target; the actor "
"ascends critic 1 only; see that entry and RL-DDPG-SHORT for the formulation, including how "
"the prior sets the initial policy and the whitened coordinates). Here one RL step "
"is the whole statistics window — exactly one standard forward evaluation, the same statistical "
"window as every other method on the leaderboard — so there is one RL update per forward "
"evaluation and each step costs 1 unit."
),
"math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q^{(1)}_\vartheta(x,F)\right]_{F=F_\theta}\right)",
"guardrails": (
"Same TD3 heuristics as RL-TD3-SHORT: twin critics with a min-target, delayed policy/target "
"updates (every 2 critic updates), and clipped target-policy smoothing noise. Same `γ=0.5<1`, "
"Polyak rate, frozen standardization and 0.1 prior-std exploration noise; the spin-up is paid "
"on every step; per-cell budget is capped at 10000 units."
),
"params": [
{"Parameter": "γ (discount)", "Value": "0.5"},
{"Parameter": "Polyak rate", "Value": "0.05"},
{"Parameter": "critics / policy delay", "Value": "2 / 2"},
{"Parameter": "target smoothing noise (clip)", "Value": "0.2 (0.5), prior-std units"},
{"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
{"Parameter": "batch size / updates per step", "Value": "64 / 1"},
{"Parameter": "steps per window / warm-up steps / buffer size", "Value": "1 / 20 / 2000"},
{"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
{"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
],
"notes": (
"**One RL update per forward evaluation, so far more expensive than RL-TD3-SHORT.** **Not tuned "
"per case.** Converges on every seed on L63 and L96-const (slower than RL-DDPG "
"on L96-const, and on L63 faster at targets 1.0–1.1 but slower at 1.2), but never reached the "
"targets on L96 flux-force or vec-force within the budget."
),
},
"IES-IEKF": {
"citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025",
"url": "https://doi.org/10.1029/2024MS004713",
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `acquire_batch`), variant `:iekf` (leaderboard key `boed-iekf`).",
"math": (
"An iterated emulate-sample loop. Iteration 1 forward-evaluates an "
"`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later "
"iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; "
"(ii) draws an approximate posterior by running an IEKF (`GaussNewtonInversion`) on the GP-mean "
"forward map — no true forward evaluations — starting from 1000 LHS prior draws; (iii) takes the next "
"`N_ens` forward-model evaluation points as a random subsample of that IEKF ensemble. The batch is "
"forward-evaluated and appended to the training set, so each iteration emulates the forward model, "
"samples the posterior from the emulator, and spends its forward runs where that posterior has mass."
),
"guardrails": (
"The posterior sampler is a fixed-step IEKF (`DefaultScheduler(0.1)`, 30 steps, total pseudo-time "
"T = 3), so it is a finite-time approximate posterior rather than a converged one; the observation "
"noise in whitened output space is the identity and the GP predictive variance is ignored. GP fits "
"keep kernel and noise hyperparameters inside bounds set relative to each mode's data scale (±2 "
"decades on length scales and signal std; noise std between 1e-3 and 1 of the signal std) to avoid "
"degenerate near-zero-noise fits, with `Fminbox` capped at 50 inner / 3 outer iterations. Inputs "
"and outputs are PCA-whitened and truncated against the prior and the observation covariance."
),
"params": [
{"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"},
{"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"},
{"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the IEKF ensemble)"},
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
{"Parameter": "retain_var (output whitening)", "Value": "0.99"},
{"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"},
],
"notes": (
"**Cheap per iteration:** no MCMC and no batch optimization, only GP-mean predictions "
"inside the IEKF. The batch is a plain draw from the posterior with no diversity criterion, so points are "
"not spread out deliberately. In the current UQ data (c = 1.5, best ensemble size) it needs a "
"mean budget of 5.0 on L63 and 5.4 on L96 const-force, but on the two high-dimensional L96 cases "
"(vec-force and NN flux-force) it fails on at least 35% and 40% of seeds, respectively, at every ensemble size."
),
},
"EKS": {
"citation": "Garbuno-Inigo, Nüsken and Reich, SIAM J. Appl. Dyn. Syst., 2020 (ALDI); Garbuno-Inigo, Hoffmann, Li and Stuart, SIAM J. Appl. Dyn. Syst., 2020 (original EKS)",
"url": "https://doi.org/10.1137/19M1304891",
"source": "`uq_experiments/KalmanSampler` — `calibrate_l63.jl`/`calibrate_l96.jl`, `experiment_config.jl` (`METHOD=sampler`, leaderboard key `sampler`); `EKP.Sampler(prior)` with EnsembleKalmanProcesses' default `sampler_type = \"aldi\"`.",
"math": (
"EKS (a.k.a ALDI): the Ensemble Kalman Sampler in its affine-invariant Langevin form (ALDI). It is a "
"derivative-free interacting-particle system that approximately samples the posterior. Each iteration "
"moves every member with an ensemble-Kalman drift (parameter/output ensemble covariances stand in for "
"a simulator Jacobian), a prior term applied implicitly, a finite-sample correction `(d+1)/J` times "
"the member's deviation from the ensemble mean, and noise built from the ensemble square root. Unlike "
"EKI the noise keeps the ensemble from collapsing, so the ensemble approximates the posterior rather "
"than a point estimate. There is no emulator: the ensemble at each iteration is itself the posterior "
"estimate."
),
"math_eq": r"\theta^{(j)}_{n+1} = \left(I + \Delta t_n C_n \Gamma_\theta^{-1}\right)^{-1}\left[\theta^{(j)}_n - \frac{\Delta t_n}{J}\sum_{k=1}^J \langle \mathcal{G}(\theta^{(k)}_n) - \bar{\mathcal{G}}_n,\, \Gamma_y^{-1}(\mathcal{G}(\theta^{(j)}_n) - y)\rangle (\theta^{(k)}_n - \bar\theta_n) + \Delta t_n C_n \Gamma_\theta^{-1} m_\theta + \Delta t_n \frac{d+1}{J}(\theta^{(j)}_n - \bar\theta_n)\right] + \sqrt{\frac{2\Delta t_n}{J}}\sum_{k=1}^J \xi_{jk}\,(\theta^{(k)}_n - \bar\theta_n)",
"guardrails": (
"The step size comes from `EKSStableScheduler()` (numerator 1, nugget `eps()`), so it is adaptive and "
"data-dependent, `Δt = 1/(‖D‖ + eps)` with `D` the ensemble-weighted data-misfit matrix, and needs "
"no hand-tuned schedule."
),
"params": [
{"Parameter": "sampler_type", "Value": "\"aldi\" (EKP default)"},
{"Parameter": "scheduler", "Value": "EKSStableScheduler() (numerator 1, nugget eps())"},
],
"notes": (
"**No emulator and no MCMC; each iteration costs `N_ens` forward runs.** In the current UQ data "
"(c = 1.5, best reliable ensemble size) it needs a mean budget of 4.2 on L63 (15% failures) and 4.2 "
"on L96 const-force, taking only about one iteration. On the two high-dimensional cases it needs "
"7 to 8 iterations, so the budget grows with ensemble size: 353 at size 50 on NN flux-force and "
"460 at size 55 on vec-force (5% failures each); the largest sizes fail on up to 25-30% of seeds."
),
},
"IES-EKS": {
"citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025; Garbuno-Inigo, Nüsken and Reich, SIAM J. Appl. Dyn. Syst., 2020 (ALDI)",
"url": "https://doi.org/10.1029/2024MS004713",
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_eks`, `acquire_batch`), variant `:eks` (leaderboard key `boed-eks`); `EKP.Sampler(prior)` with EnsembleKalmanProcesses' default `sampler_type = \"aldi\"`.",
"math": (
"The same iterated emulate-sample loop as IES-IEKF and IES-TMCMC (see those entries), with the "
"posterior sampler swapped for the Ensemble Kalman Sampler (EKP `Sampler`). Iteration 1 "
"forward-evaluates an `N_ens`-point Latin-hypercube design in the truncated, prior-whitened space. "
"Each later iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs so far; "
"(ii) draws an approximate posterior by running the sampler on the GP-mean forward map (no true "
"forward evaluations), from a 200-member Latin-hypercube standard-normal start with the prior "
"N(0, I) and identity noise; (iii) takes the next `N_ens` forward-model evaluation points as a random "
"subsample of that ensemble."
),
"guardrails": (
"The sampler is the default `EKP.Sampler(prior)`, i.e. `sampler_type = \"aldi\"` (the same ALDI "
"variant used by the EKS and CES-EKS entries), driven by "
"`EKSStableScheduler(alpha = 0.25)` for a fixed 500 steps. `alpha = 0.25` was chosen because it "
"sat closer to the ST-MCMC reference than the default 1, and `alpha >= 3` is unstable. Like the IEKF "
"variant it ignores the GP predictive variance. GP fits use the same data-relative hyperparameter "
"bounds and iteration caps as IES-IEKF. The posterior has 200 draws rather than 1000; the "
"shortfall is padded with NaN rows that the leaderboard stage drops."
),
"params": [
{"Parameter": "sampler_type", "Value": "\"aldi\" (EKP default, Sampler(prior))"},
{"Parameter": "EKS particles (eks_n_particles)", "Value": "200"},
{"Parameter": "scheduler", "Value": "EKSStableScheduler(alpha = 0.25)"},
{"Parameter": "steps (eks_max_iters / eks_T)", "Value": "500 / Inf (fixed step count)"},
{"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the EKS ensemble)"},
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
],
"notes": (
"**Cheap in forward runs but with a fixed 500-step sampler cost per iteration** (GP-mean evaluations "
"only). In the current UQ data (c = 1.5, best ensemble size) its mean budget is 5.4 on L63, 4.2 on "
"L96 const-force and 112 on NN flux-force (size 80), all with 0% failures. On vec-force it fails on "
"at least 20% of seeds at every ensemble size, so it is not suitable there under the 20% threshold."
),
},
"GBOED-EKS": {
"citation": "Holthuijzen et al. 2026 (EIG acquisition); Garbuno-Inigo, Nüsken and Reich, SIAM J. Appl. Dyn. Syst., 2020 (ALDI posterior sampler)",
"url": "https://arxiv.org/abs/2508.13071",
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_eks`, `optimize_batch`, `acquire_batch`), variant `:eks_eig` (leaderboard key `boed-eks-eig`); `EKP.Sampler(prior)` with EnsembleKalmanProcesses' default `sampler_type = \"aldi\"`.",
"math": (
"GBOED with the posterior sampler swapped for the Ensemble Kalman Sampler (see the GBOED-IEKF and "
"IES-EKS entries). Iteration 1 forward-evaluates an `N_ens`-point Latin-hypercube design in the "
"truncated prior-whitened space. Each later iteration: (i) refits one GP per whitened output mode on "
"all `(θ, G(θ))` pairs so far; (ii) draws an approximate posterior by running the sampler on the "
"GP-mean forward map (no true forward evaluations); (iii) chooses the next `N_ens` evaluation "
"points by jointly maximizing the closed-form Expected Information Gain (EIG) of the whole batch "
"against a 50-member random subsample of the EKS ensemble, starting from a random subsample of "
"the full ensemble."
),
"math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}",
"guardrails": (
"The sampler is the default `EKP.Sampler(prior)` with `sampler_type = \"aldi\"` (as in the EKS entry), run for a "
"fixed 500 steps of `EKSStableScheduler(alpha = 0.25)` with 200 particles, ignoring the GP "
"predictive variance. The batch is optimized with `Optim.jl`'s `LBFGS` inside the χ² ball of the "
"prior-whitened space (radial `tanh` squash, so every candidate has radius below √χ²_k(0.999)), with "
"a hard cap on raw EIG evaluations (`eig_call_limit`) and an outer-iteration cap "
"(`eig_outer_iters`), logging a warning that the returned batch may be \"under-optimized\" if either "
"is hit; a diagonal jitter (`eig_jitter`, relative to the GP signal variance) stabilizes the "
"covariance solves and log-determinants. GP fits use the same data-relative bounds and iteration "
"caps as IES-IEKF."
),
"params": [
{"Parameter": "sampler_type", "Value": "\"aldi\" (EKP default, Sampler(prior))"},
{"Parameter": "EKS particles (eks_n_particles)", "Value": "200"},
{"Parameter": "scheduler", "Value": "EKSStableScheduler(alpha = 0.25)"},
{"Parameter": "steps (eks_max_iters / eks_T)", "Value": "500 / Inf (fixed step count)"},
{"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"},
{"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"},
{"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"},
{"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
{"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"},
{"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
{"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
],
"notes": (
"**Adds the EIG batch optimization to IES-EKS**, so it pays for the inner EIG-maximization loop on "
"top of the 500-step sampler; hitting its call or outer-iteration limits gives \"under-optimized\" "
"batches. In the current UQ data (c = 1.5, best ensemble size) its mean budget is 5.2 on L63, 4.2 on "
"L96 const-force and 105 on NN flux-force (size 80, 5% failures). On vec-force it fails on at least "
"25% of seeds at every ensemble size, so it is not suitable there under the 20% failure threshold."
),
},
"GBOED-IEKF": {
"citation": "Holthuijzen et al. 2026 (EIG acquisition); IEKF posterior sampler replaces ST-MCMC",
"url": "https://arxiv.org/abs/2508.13071",
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_iekf`, `optimize_batch`, `acquire_batch`), variant `:iekf_eig` (leaderboard key `boed-iekf-eig`).",
"math": (
"GBOED with the posterior sampler swapped for an IEKF. Iteration 1 forward-evaluates an `N_ens`-point "
"Latin-hypercube design in the truncated prior-whitened space. Each later iteration: (i) refits one GP "
"per whitened output mode on all `(θ, G(θ))` pairs so far; (ii) draws an approximate posterior by "
"running an IEKF (`GaussNewtonInversion`) on the GP-mean forward map — no true forward evaluations; "
"(iii) chooses the next `N_ens` evaluation points by jointly maximizing the closed-form Expected "
"Information Gain (EIG) of the whole batch against a 50-member random subsample X' of the IEKF "
"ensemble, starting from a random subsample of the full ensemble."
),
"math_eq": r"\mathrm{EIG}(X) = \frac{1}{k}\sum_j \frac{1}{2}\log\frac{\det K_{pp}^{(j)}}{\det \Sigma'^{(j)}}, \quad \Sigma'^{(j)} = K_{pp}^{(j)} - K_{pc}^{(j)}\left(K_{cc}^{(j)}\right)^{-1}K_{cp}^{(j)}",
"guardrails": (
"The IEKF posterior is a fixed-step, finite-time approximation (`DefaultScheduler(0.1)`, 30 steps, "
"T = 3) that ignores the GP predictive variance. The batch is optimized with `Optim.jl`'s "
"`LBFGS` inside the χ² ball of the prior-whitened space (radial `tanh` squash, so every candidate "
"has radius below √χ²_k(0.999)), with a hard cap on raw EIG evaluations (`eig_call_limit`) and an "
"outer-iteration cap (`eig_outer_iters`) enforced manually, logging a warning that the returned "
"batch may be \"under-optimized\" if either is hit; a diagonal jitter (`eig_jitter`, relative to the "
"GP signal variance) stabilizes the covariance solves and log-determinants. GP fits use the same "
"data-relative hyperparameter bounds and iteration caps as IES-IEKF."
),
"params": [
{"Parameter": "IEKF ensemble size (n_posterior_samples)", "Value": "1000"},
{"Parameter": "IEKF step size / steps", "Value": "0.1 / 30 (T = 3)"},
{"Parameter": "n_eig_posterior_samples (EIG subsample X')", "Value": "50"},
{"Parameter": "eig_region / eig_ball_quantile", "Value": ":ball / 0.999 (χ² ball in prior-whitened space)"},
{"Parameter": "eig_optim_iters / eig_outer_iters", "Value": "500 / 20"},
{"Parameter": "eig_call_limit", "Value": "2000 EIG evaluations"},
{"Parameter": "eig_g_tol / eig_f_reltol", "Value": "1e-3 / 1e-6"},
{"Parameter": "eig_jitter", "Value": "1e-6 × GP signal variance"},
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
{"Parameter": "batch_init_strategy", "Value": ":posterior_subsample"},
{"Parameter": "retain_var / retain_var_input", "Value": "0.99 / 0.99 (L63, L96 const), 0.9 (vec, flux)"},
],
"notes": (
"**Adds the EIG batch optimization to IES-IEKF while keeping the cheap IEKF posterior**, so it avoids "
"ST-MCMC cost but still pays for the inner EIG-maximization loop, which costs ~n³ per output mode in "
"the subsample size and slows every iteration as the cumulative GP training set grows; hitting its "
"call or outer-iteration limits gives \"under-optimized\" batches. In the current UQ data (c = 1.5, "
"best ensemble size) it has the lowest mean budget of the four variants on L63 (4.8) and ties "
"GBOED-TMCMC on L96 const-force (4.4). On the two high-dimensional L96 cases it has low budgets "
"(55 on vec-force, 50 on NN flux-force) but fails on at least 35% and 40% of seeds, respectively, "
"at every ensemble size, so it is not suitable there under the 20% failure threshold."
),
},
"IES-TMCMC": {
"citation": "Elsaesser et al., J. Adv. Model. Earth Syst., 2025",
"url": "https://doi.org/10.1029/2024MS004713",
"source": "`uq_experiments/BayesianOptimalExperimentalDesign` — `boed_core.jl` (`run_tmcmc`, `boed_loglik`, `acquire_batch`), variant `:tmcmc` (leaderboard key `boed-tmcmc`).",
"math": (
"An iterated emulate-sample loop. Iteration 1 forward-evaluates an "
"`N_ens`-point Latin-hypercube design in the truncated, prior-whitened parameter space. Each later "
"iteration: (i) refits one GP per whitened output mode on all `(θ, G(θ))` pairs collected so far; "
"(ii) draws posterior samples from the GP-based posterior via Transitional/Sequential MCMC "
"(Ching & Chen, 2007), using the GP-marginal Gaussian likelihood below; (iii) takes the next `N_ens` "
"forward-model evaluation points as a random subsample of those posterior draws. The batch is "
"forward-evaluated and appended to the training set, so each iteration emulates the forward model, "
"samples the posterior from the emulator, and spends its forward runs where that posterior has mass."
),
"math_eq": r"\log p(y\mid z) = \sum_j \left[-\frac{(\mu_j(z)-y_j)^2}{2\,(1+\sigma_j^2(z))} - \frac{1}{2}\log\left(2\pi\,(1+\sigma_j^2(z))\right)\right]",
"guardrails": (
"The likelihood includes the GP predictive variance `σ_j²(z)` (the whitened observation noise is the "
"identity), so the posterior widens where the surrogate is uncertain, and keeps the normalizing "
"log(2π·v) term because ST-MCMC needs a properly normalized density. TMCMC runs with burn-in and "
"thinning below its own 20 / 3 defaults since they multiply per-stage cost. GP fits keep "
"hyperparameters inside data-relative bounds (±2 decades on length scales and signal std; noise std "
"between 1e-3 and 1 of the signal std) to avoid the near-zero-noise fits that ST-MCMC resampling "
"can collapse onto, with `Fminbox` capped at 50 inner / 3 outer iterations."
),
"params": [
{"Parameter": "n_posterior_samples (TMCMC population)", "Value": "1000"},
{"Parameter": "tmcmc_burnin / tmcmc_thin", "Value": "5 / 1 (below TMCMC's 20 / 3 defaults)"},
{"Parameter": "batch size per iteration", "Value": "N_ens (random subsample of the TMCMC draws)"},
{"Parameter": "gp_optim_iters / gp_outer_iters", "Value": "50 / 3"},
{"Parameter": "retain_var (output whitening)", "Value": "0.99"},
{"Parameter": "retain_var_input (input whitening)", "Value": "0.99 (L63, L96 const) / 0.9 (vec, flux)"},
],
"notes": (
"**The batch is a plain draw from the ST-MCMC posterior, with no batch optimization.** TMCMC cost grows roughly linearly with "
"the population size, and a small population gives a degenerate posterior in the high-dimensional "
"L96 cases. In the current UQ data (c = 1.5, best ensemble size) its mean budget is 6.4 on L63 and 5.6 on L96 "
"const-force, but on L96 vec-force it is 240 and it fails on at least 85% of seeds at every ensemble size."
),
},
}
# KNOWN_METHODS is the source of truth for the abbreviation/full-name list,
# so this page renders even without any NetCDF data loaded.
abbrs = sorted(meta["abbreviation"] for meta in KNOWN_METHODS.values())
full_name_by_abbr = {meta["abbreviation"]: meta["Method"] for meta in KNOWN_METHODS.values()}
# Selection UI (defaults to query param if valid)
abbr = st.query_params.get("method")
default_idx = abbrs.index(abbr) if isinstance(abbr, str) and abbr in abbrs else 0
sel = str(st.selectbox("Choose a method", options=abbrs, index=default_idx))
# Persist selection to URL
st.query_params["method"] = sel
# Display details
meta = method_meta.get(sel, {})
st.subheader(f"{sel} — {full_name_by_abbr.get(sel, '')}")
if meta.get("source"):
st.caption(meta["source"])
st.markdown("**Update Equation**")
st.markdown(meta.get("math", "Mathematical form pending"))
if meta.get("math_eq"):
st.latex(meta["math_eq"])
st.markdown("**Robustness, Speed & Guardrails**")
st.markdown(meta.get("guardrails", "Guardrail details pending"))
st.markdown("**Configuration Parameters**")
params = meta.get("params")
if params:
st.dataframe(pd.DataFrame(params), hide_index=True, use_container_width=True)
else:
st.write("Parameter details pending")
st.markdown("**Convergence & Speed Notes**")
st.markdown(meta.get("notes", "No notes recorded yet"))
col1, col2 = st.columns(2)
with col1:
with st.container(border=True):
st.markdown("**Citation**")
st.write(meta.get("citation", "Citation pending"))
with col2:
with st.container(border=True):
st.markdown("**Link**")
st.link_button("Open reference", meta.get("url", "https://example.com"))
st.page_link("pages/OptimizationLeaderboard.py", label="⬅ Back to Optimization Leaderboard", icon="↩️")
|