Reinforcement Learning (Actor Critic)

.cache/known_methods_snapshot.json

@@ -4,7 +4,7 @@
       "abbreviation": "TEKI",
       "Method": "Tikhonov Regularized Ensemble Kalman Inversion",
       "parallelism": "parallel-interacting",
-      "update_type": "kalman",
+      "update_type": "gradient (Kalman)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -15,7 +15,7 @@
       "abbreviation": "ETKI",
       "Method": "Ensemble Transform Kalman Inversion",
       "parallelism": "parallel-interacting",
-      "update_type": "kalman",
+      "update_type": "gradient (Kalman)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -26,7 +26,7 @@
       "abbreviation": "IEKF",
       "Method": "Iterative Ensemble Kalman Filter",
       "parallelism": "parallel-interacting",
-      "update_type": "kalman",
+      "update_type": "gradient (Kalman)",
       "method_goal": "uq",
       "emulator_use": "none",
       "aliases": [
@@ -39,7 +39,7 @@
       "abbreviation": "UKI",
       "Method": "Unscented Kalman Inversion",
       "parallelism": "parallel-interacting",
-      "update_type": "kalman",
+      "update_type": "gradient (Kalman)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -72,7 +72,7 @@
       "abbreviation": "CES-EKI-DMC",
       "Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
       "parallelism": "parallel-interacting",
-      "update_type": "kalman",
+      "update_type": "gradient (Kalman)",
       "method_goal": "uq",
       "emulator_use": "after-optimize",
       "aliases": [
@@ -83,7 +83,7 @@
       "abbreviation": "CES-EKI-CONST",
       "Method": "Calibrate Emulate Sample (EKI-Constant Scheduler)",
       "parallelism": "parallel-interacting",
-      "update_type": "kalman",
+      "update_type": "gradient (Kalman)",
       "method_goal": "uq",
       "emulator_use": "after-optimize",
       "aliases": [
@@ -94,7 +94,7 @@
       "abbreviation": "CES-IEKF-CONST",
       "Method": "Calibrate Emulate Sample (IEKF-Constant Scheduler)",
       "parallelism": "parallel-interacting",
-      "update_type": "kalman",
+      "update_type": "gradient (Kalman)",
       "method_goal": "uq",
       "emulator_use": "after-optimize",
       "aliases": [
@@ -105,7 +105,7 @@
       "abbreviation": "ADAM",
       "Method": "Adaptive Moment Estimation",
       "parallelism": "serial",
-      "update_type": "gradient",
+      "update_type": "gradient (exact)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -116,7 +116,7 @@
       "abbreviation": "LM",
       "Method": "Levenberg-Marquardt",
       "parallelism": "serial",
-      "update_type": "gradient",
+      "update_type": "gradient (exact)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -142,7 +142,7 @@
       "abbreviation": "LM-KGMM",
       "Method": "Score-based Levenberg-Marquardt (KGMM)",
       "parallelism": "serial",
-      "update_type": "gradient",
+      "update_type": "gradient (score)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -155,7 +155,7 @@
       "abbreviation": "LM-DSM",
       "Method": "Score-based Levenberg-Marquardt (DSM)",
       "parallelism": "serial",
-      "update_type": "gradient",
+      "update_type": "gradient (score)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -168,7 +168,7 @@
       "abbreviation": "ADAM-KGMM",
       "Method": "Score-based Adam (KGMM)",
       "parallelism": "serial",
-      "update_type": "gradient",
+      "update_type": "gradient (score)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -181,7 +181,7 @@
       "abbreviation": "ADAM-DSM",
       "Method": "Score-based Adam (DSM)",
       "parallelism": "serial",
-      "update_type": "gradient",
+      "update_type": "gradient (score)",
       "method_goal": "optimization",
       "emulator_use": "none",
       "aliases": [
@@ -189,6 +189,52 @@
         "score-based adam (gfdt, dsm) (extended window)",
         "score-based adam (gfdt, dsm) (parallel branches)"
       ]
+    },
+    "rl-ddpg-short": {
+      "abbreviation": "RL-DDPG-SHORT",
+      "Method": "Reinforcement Learning (DDPG, short RL step)",
+      "parallelism": "serial",
+      "update_type": "gradient (score)",
+      "method_goal": "optimization",
+      "emulator_use": "none",
+      "aliases": [
+        "rl-ddpg-short"
+      ]
+    },
+    "rl-ddpg": {
+      "abbreviation": "RL-DDPG",
+      "Method": "Reinforcement Learning (DDPG)",
+      "parallelism": "serial",
+      "update_type": "gradient (score)",
+      "method_goal": "optimization",
+      "emulator_use": "none",
+      "aliases": [
+        "rl-ddpg",
+        "rl-ddpg-long"
+      ]
+    },
+    "rl-td3-short": {
+      "abbreviation": "RL-TD3-SHORT",
+      "Method": "Reinforcement Learning (TD3, short RL step)",
+      "parallelism": "serial",
+      "update_type": "gradient (score)",
+      "method_goal": "optimization",
+      "emulator_use": "none",
+      "aliases": [
+        "rl-td3-short"
+      ]
+    },
+    "rl-td3": {
+      "abbreviation": "RL-TD3",
+      "Method": "Reinforcement Learning (TD3)",
+      "parallelism": "serial",
+      "update_type": "gradient (score)",
+      "method_goal": "optimization",
+      "emulator_use": "none",
+      "aliases": [
+        "rl-td3",
+        "rl-td3-long"
+      ]
     }
   },
   "observed_methods": [
@@ -200,6 +246,10 @@
     "lm",
     "lm-dsm",
     "lm-kgmm",
+    "rl-ddpg",
+    "rl-ddpg-short",
+    "rl-td3",
+    "rl-td3-short",
     "teki",
     "uki"
   ],

.claude/skills/leaderboard-manager/references/add-method.md

@@ -45,7 +45,7 @@ at line 7). Follow the existing pattern exactly:
     "abbreviation": "NEWABBR",           # Short uppercase display name
     "Method": "Full Method Name",        # Human-readable name for tables
     "parallelism": "parallel-interacting",  # serial | parallel-independent | parallel-interacting
-    "update_type": "kalman",             # gradient | kalman | general
+    "update_type": "gradient (Kalman)",  # gradient (exact) | gradient (Kalman) | gradient (score) | general
     "method_goal": "optimization",       # optimization | uq
     "emulator_use": "none",              # none | within-optimize | after-optimize
     "aliases": ["new-key", "alias2"],    # All raw strings that may appear in NetCDF algorithm_type

.claude/skills/leaderboard-manager/references/populate-metadata.md

@@ -114,7 +114,7 @@ short label that appears in table columns), edit `KNOWN_METHODS` in
     "abbreviation": "UKI",           # Short label in table columns
     "Method": "Unscented Kalman Inversion",  # Full name in Method column
     "parallelism": "parallel-interacting",   # serial | parallel-independent | parallel-interacting
-    "update_type": "kalman",                 # gradient | kalman | general
+    "update_type": "gradient (Kalman)",      # gradient (exact) | gradient (Kalman) | gradient (score) | general
     "method_goal": "optimization",            # optimization | uq
     "emulator_use": "none",                   # none | within-optimize | after-optimize
     "aliases": ["uki"],              # Raw algorithm_type strings from NetCDF

CLAUDE.md

@@ -78,6 +78,8 @@ calibration_benchmark/
 │   ├── opt_dsm_score_lm/        Score-based LM (DSM score estimator) results (per-benchmark files)
 │   ├── opt_kgmm_score_adam/     Score-based ADAM (KGMM score estimator) results (per-benchmark files)
 │   ├── opt_dsm_score_adam/      Score-based ADAM (DSM score estimator) results (per-benchmark files)
+│   ├── rl-td3/                  RL-TD3-SHORT / RL-TD3 (TD3 variant of the RL experiments; RL-TD3 files are `rl-td3-long` in the NetCDF) results (per-benchmark files)
+│   ├── rl-ddpg/                 RL-DDPG-SHORT / RL-DDPG (DDPG actor-critic; short RL step / standard-window RL step; RL-DDPG files are `rl-ddpg-long` in the NetCDF) results (per-benchmark files)
 │   ├── ces-eki-dmc_results/     CES-EKI-DMC ensemble-results files (UQ source)
 │   ├── gboed-uq_results/        GBOED ensemble-results files (UQ source)
 │   ├── gnki-uq_results/         IEKF (a.k.a GNKI) ensemble-results files (UQ source)
@@ -161,13 +163,18 @@ Four independent tags on every `KNOWN_METHODS` entry, each scoped to its own axi
 
 - **`parallelism`** — how the search explores parameter space:
   `serial` (single point estimate advanced step by step, e.g. `ADAM`/`LM`) ·
+  `serial (short window)` (serial, but each step advances a sliding statistics window by a
+  fraction of its length rather than a full window, e.g. `RL-DDPG-SHORT`/`RL-TD3-SHORT`) ·
   `parallel-independent` (population updated with no coupling between members,
   e.g. `ABC`'s accepted samples / `HM`'s per-wave resampling) ·
   `parallel-interacting` (ensemble members coupled through a shared update each
   iteration — any Kalman-based method).
 - **`update_type`** — the mechanism driving each update step:
-  `gradient` (follows the loss gradient or a Gauss-Newton approximation) ·
-  `kalman` (a possibly linearized/unscented Kalman-style ensemble update) ·
+  `gradient (exact)` (loss gradient / Gauss-Newton step differentiated through the forward
+  model, e.g. `ADAM`/`LM`) ·
+  `gradient (Kalman)` (a possibly linearized/unscented Kalman-style ensemble update) ·
+  `gradient (score)` (gradient built from an estimated score or learned critic rather than a
+  simulator Jacobian, e.g. `*-KGMM`, `*-DSM`, `RL-*`) ·
   `general` (anything else, e.g. `ABC`'s rejection sampling, `HM`'s implausibility cuts).
 - **`method_goal`** — what the method is built to report:
   `optimization` (single best-fit parameter estimate) or `uq` (full posterior /
@@ -179,7 +186,7 @@ Four independent tags on every `KNOWN_METHODS` entry, each scoped to its own axi
   `after-optimize` (fits a surrogate once, after calibration finishes, e.g. `CES-EKI-DMC`).
 
 `update_type` deliberately avoids a "Bayesian" label — Kalman methods are approximate
-Bayesian updates too, so the split is about mechanism (gradient vs. Kalman vs. general),
+Bayesian updates too, so the split is about mechanism (exact vs. Kalman vs. score gradient vs. general),
 not philosophy. `_UPDATE_TYPE_ORDER` in `src/common/leaderboard.py` controls
 table/suitability sort order and must stay in sync with `update_type` values.
 A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
@@ -188,20 +195,24 @@ A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
 
 | Canonical | Abbreviation | parallelism | update_type | method_goal | emulator_use | Aliases |
 |---|---|---|---|---|---|---|
-| `teki` | TEKI | parallel-interacting | kalman | optimization | none | — |
-| `etki` | ETKI | parallel-interacting | kalman | optimization | none | — |
-| `iekf` | IEKF | parallel-interacting | kalman | uq | none | gnsl, gnki |
-| `uki` | UKI | parallel-interacting | kalman | optimization | none | — |
+| `teki` | TEKI | parallel-interacting | gradient (Kalman) | optimization | none | — |
+| `etki` | ETKI | parallel-interacting | gradient (Kalman) | optimization | none | — |
+| `iekf` | IEKF | parallel-interacting | gradient (Kalman) | uq | none | gnsl, gnki |
+| `uki` | UKI | parallel-interacting | gradient (Kalman) | optimization | none | — |
 | `abc` | ABC | parallel-independent | general | uq | none | — |
 | `hm` | HM | parallel-independent | general | uq | within-optimize | — |
-| `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | kalman | uq | after-optimize | — |
-| `adam` | ADAM | serial | gradient | optimization | none | — |
-| `lm` | LM | serial | gradient | optimization | none | gradient_descent |
+| `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | gradient (Kalman) | uq | after-optimize | — |
+| `adam` | ADAM | serial | gradient (exact) | optimization | none | — |
+| `lm` | LM | serial | gradient (exact) | optimization | none | gradient_descent |
 | `gboed` | GBOED | parallel-interacting | general | uq | within-optimize | boed |
-| `lm-kgmm` | LM-KGMM | serial | gradient | optimization | none | — |
-| `lm-dsm` | LM-DSM | serial | gradient | optimization | none | — |
-| `adam-kgmm` | ADAM-KGMM | serial | gradient | optimization | none | — |
-| `adam-dsm` | ADAM-DSM | serial | gradient | optimization | none | — |
+| `lm-kgmm` | LM-KGMM | serial | gradient (score) | optimization | none | — |
+| `lm-dsm` | LM-DSM | serial | gradient (score) | optimization | none | — |
+| `adam-kgmm` | ADAM-KGMM | serial | gradient (score) | optimization | none | — |
+| `adam-dsm` | ADAM-DSM | serial | gradient (score) | optimization | none | — |
+| `rl-ddpg-short` | RL-DDPG-SHORT | serial (short window) | gradient (score) | optimization | none | — |
+| `rl-ddpg` | RL-DDPG | serial | gradient (score) | optimization | none | rl-ddpg-long |
+| `rl-td3-short` | RL-TD3-SHORT | serial (short window) | gradient (score) | optimization | none | — |
+| `rl-td3` | RL-TD3 | serial | gradient (score) | optimization | none | rl-td3-long |
 
 ## How to extend
 

data/rl-ddpg/leaderboard_rl-ddpg-long_l63_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75420eb82ea3b59b30f803fc289a85af15869a782bc1d74ff8ed899d6997abce
+size 9200

data/rl-ddpg/leaderboard_rl-ddpg-long_l96_const-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85660a09db96d4486bdfec01c8310b2fa8e888903e930437334f678c2a24f693
+size 9200

data/rl-ddpg/leaderboard_rl-ddpg-long_l96_flux-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d541f2a8bb5ed357bd6a47183546f95f243202afc6a47ac64a4c78d5b1ae1b0b
+size 9200

data/rl-ddpg/leaderboard_rl-ddpg-long_l96_vec-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d541f2a8bb5ed357bd6a47183546f95f243202afc6a47ac64a4c78d5b1ae1b0b
+size 9200

data/rl-ddpg/leaderboard_rl-ddpg-short_l63_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68189692e5ecab696cd87030be1fcd94a4c018f6035c2e361924dfa1b1904859
+size 9200

data/rl-ddpg/leaderboard_rl-ddpg-short_l96_const-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f461cf02963eccb7fefb4d41c0451dd823232b93ae1b5cfa971df9e2ffbd8d9
+size 9200

data/rl-ddpg/leaderboard_rl-ddpg-short_l96_flux-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cec229bfc4948a5c2ef382fba6b733221f8ef5ea3143e5b72be9c93f833e512
+size 9200

data/rl-ddpg/leaderboard_rl-ddpg-short_l96_vec-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce5cff4a54bcff60d862a0f8f29d956cc9d6eba230f3469992a71fd6e44a7efa
+size 9200

data/rl-td3/leaderboard_rl-td3-long_l63_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f546837f79d3f020fa8e8c6f1f97f4aa9543dda0631d8a5b462687ff49793697
+size 9200

data/rl-td3/leaderboard_rl-td3-long_l96_const-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27956dd2b55fe624f191cd1759701d247f6efc09cd833daa5adc1851c6c0c42d
+size 9200

data/rl-td3/leaderboard_rl-td3-long_l96_flux-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0562fc2dc63bc374389772c9ebba5fa7118edb7217e60a04e02a246b671337b0
+size 9200

data/rl-td3/leaderboard_rl-td3-long_l96_vec-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c51d0d65cdb3abf2f3f095d790a028132039eb2c7c161c5544fc962ba615196
+size 9200

data/rl-td3/leaderboard_rl-td3-short_l63_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c33c3864a00e00e57baa9f8d08521c730da84695c0c1139d6ba60e3e2b48d0e
+size 9200

data/rl-td3/leaderboard_rl-td3-short_l96_const-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abd015cc5749e5a2db779756192a4eb1521c01170a69fb551b52b65dedfb71a2
+size 9200

data/rl-td3/leaderboard_rl-td3-short_l96_flux-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:599281064f3f993989a9f93c1eb24bb6a1e30dba7c5b900da86ac90ba0d99a1b
+size 9200

data/rl-td3/leaderboard_rl-td3-short_l96_vec-force_2026-10-04.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80282ab8703b8addb2114fde70b5d30934b0e6f5b04260cf092d69eef515f8b9
+size 9200

src/common/leaderboard.py

@@ -53,9 +53,10 @@ def _method_color(present_abbrevs: list[str]) -> alt.Color:
 # Unknown/unmapped update types fall back to 99 and appear at the end.
 # See method_registry.py for the full four-axis taxonomy.
 _UPDATE_TYPE_ORDER: dict[str, int] = {
-    "kalman": 0,
-    "gradient": 1,
-    "general": 2,
+    "gradient (Kalman)": 0,
+    "gradient (exact)": 1,
+    "gradient (score)": 2,
+    "general": 3,
 }
 
 # Failure-rate threshold (%) below which a method's runs at a given target are
@@ -119,7 +120,11 @@ def _render_suitability_table(
                 continue
             qualifying = sub[sub["failure_rate"] < failure_threshold].dropna(subset=["metric"])
             if qualifying.empty:
-                cell_text[method][bm]  = "failed"
+                # Report the best (lowest) failure rate over ensemble sizes
+                best_fail = sub["failure_rate"].min()
+                cell_text[method][bm]  = (
+                    f"failed ({best_fail:.0f}%)" if pd.notna(best_fail) else "failed"
+                )
                 cell_color[method][bm] = _UNSUITABLE
                 continue
             method_best = float(qualifying["metric"].min())
@@ -157,7 +162,9 @@ def _render_suitability_table(
         f"Evaluated at target = {suitability_target}. "
         f"**Green**: at some ensemble size, failure rate < {failure_threshold:.0f}% "
         f"and mean budget ≤ {ratio_threshold:.0f}× the best method (ratio shown). "
-        "**Red** (failed): data present but all runs failed to reach the target. "
+        "**Red** (failed): data present but no ensemble size met the criteria; "
+        "the percentage is the lowest failure rate (share of runs that did not reach the target) "
+        "across ensemble sizes. "
         "**Gray (—)**: no data for this benchmark."
     )
     st.dataframe(

src/common/method_registry.py

@@ -10,6 +10,8 @@ from pathlib import Path
 #
 # ``parallelism`` — how the search explores parameter space:
 #   "serial"               — a single point estimate advanced step by step (ADAM, LM).
+#   "serial (short window)" — serial, but each step advances a sliding statistics window by
+#                             a fraction of its length (RL-*-SHORT) rather than a full window.
 #   "parallel-independent" — a population of candidates updated with no coupling
 #                             between members (ABC's accepted samples, HM's per-wave
 #                             resampling from the non-implausible region).
@@ -17,9 +19,13 @@ from pathlib import Path
 #                             update each iteration (any Kalman-based method).
 #
 # ``update_type`` — the mechanism driving each update step:
-#   "gradient" — follows the loss gradient or a Gauss-Newton approximation of it.
-#   "kalman"   — a (possibly linearized/unscented) Kalman-style ensemble update.
-#   "general"  — anything else (e.g. ABC's rejection sampling, HM's implausibility cuts).
+#   "gradient (exact)"  — follows the loss gradient or a Gauss-Newton approximation of it,
+#                          differentiated through the forward model (ADAM, LM).
+#   "gradient (Kalman)" — a (possibly linearized/unscented) Kalman-style ensemble update,
+#                          i.e. an ensemble-estimated gradient/Gauss-Newton step.
+#   "gradient (score)"  — a gradient built from a learned/estimated score or critic rather
+#                          than a simulator Jacobian (*-KGMM, *-DSM, RL-*).
+#   "general"           — anything else (e.g. ABC's rejection sampling, HM's implausibility cuts).
 #
 # ``method_goal`` — what the method is built to report:
 #   "optimization" — a single best-fit parameter estimate.
@@ -39,7 +45,7 @@ KNOWN_METHODS = {
         "abbreviation": "TEKI",
         "Method": "Tikhonov Regularized Ensemble Kalman Inversion",
         "parallelism": "parallel-interacting",
-        "update_type": "kalman",
+        "update_type": "gradient (Kalman)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": ["teki"],
@@ -48,7 +54,7 @@ KNOWN_METHODS = {
         "abbreviation": "ETKI",
         "Method": "Ensemble Transform Kalman Inversion",
         "parallelism": "parallel-interacting",
-        "update_type": "kalman",
+        "update_type": "gradient (Kalman)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": ["etki"],
@@ -57,7 +63,7 @@ KNOWN_METHODS = {
         "abbreviation": "IEKF",
         "Method": "Iterative Ensemble Kalman Filter",
         "parallelism": "parallel-interacting",
-        "update_type": "kalman",
+        "update_type": "gradient (Kalman)",
         "method_goal": "uq",
         "emulator_use": "none",
         "aliases": ["iekf", "gnsl", "gnki"],
@@ -66,7 +72,7 @@ KNOWN_METHODS = {
         "abbreviation": "UKI",
         "Method": "Unscented Kalman Inversion",
         "parallelism": "parallel-interacting",
-        "update_type": "kalman",
+        "update_type": "gradient (Kalman)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": ["uki"],
@@ -93,7 +99,7 @@ KNOWN_METHODS = {
         "abbreviation": "CES-EKI-DMC",
         "Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
         "parallelism": "parallel-interacting",
-        "update_type": "kalman",
+        "update_type": "gradient (Kalman)",
         "method_goal": "uq",
         "emulator_use": "after-optimize",
         "aliases": ["ces-eki-dmc"]
@@ -102,7 +108,7 @@ KNOWN_METHODS = {
         "abbreviation": "CES-EKI-CONST",
         "Method": "Calibrate Emulate Sample (EKI-Constant Scheduler)",
         "parallelism": "parallel-interacting",
-        "update_type": "kalman",
+        "update_type": "gradient (Kalman)",
         "method_goal": "uq",
         "emulator_use": "after-optimize",
         "aliases": ["ces-eki-const"]
@@ -111,7 +117,7 @@ KNOWN_METHODS = {
         "abbreviation": "CES-IEKF-CONST",
         "Method": "Calibrate Emulate Sample (IEKF-Constant Scheduler)",
         "parallelism": "parallel-interacting",
-        "update_type": "kalman",
+        "update_type": "gradient (Kalman)",
         "method_goal": "uq",
         "emulator_use": "after-optimize",
         "aliases": ["ces-iekf-const"]
@@ -120,7 +126,7 @@ KNOWN_METHODS = {
         "abbreviation": "ADAM",
         "Method": "Adaptive Moment Estimation",
         "parallelism": "serial",
-        "update_type": "gradient",
+        "update_type": "gradient (exact)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": ["adam"],
@@ -129,7 +135,7 @@ KNOWN_METHODS = {
         "abbreviation": "LM",
         "Method": "Levenberg-Marquardt",
         "parallelism": "serial",
-        "update_type": "gradient",
+        "update_type": "gradient (exact)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
@@ -147,7 +153,7 @@ KNOWN_METHODS = {
         "abbreviation": "LM-KGMM",
         "Method": "Score-based Levenberg-Marquardt (KGMM)",
         "parallelism": "serial",
-        "update_type": "gradient",
+        "update_type": "gradient (score)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": [
@@ -160,7 +166,7 @@ KNOWN_METHODS = {
         "abbreviation": "LM-DSM",
         "Method": "Score-based Levenberg-Marquardt (DSM)",
         "parallelism": "serial",
-        "update_type": "gradient",
+        "update_type": "gradient (score)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": [
@@ -173,7 +179,7 @@ KNOWN_METHODS = {
         "abbreviation": "ADAM-KGMM",
         "Method": "Score-based Adam (KGMM)",
         "parallelism": "serial",
-        "update_type": "gradient",
+        "update_type": "gradient (score)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": [
@@ -186,7 +192,7 @@ KNOWN_METHODS = {
         "abbreviation": "ADAM-DSM",
         "Method": "Score-based Adam (DSM)",
         "parallelism": "serial",
-        "update_type": "gradient",
+        "update_type": "gradient (score)",
         "method_goal": "optimization",
         "emulator_use": "none",
         "aliases": [
@@ -195,6 +201,42 @@ KNOWN_METHODS = {
             "score-based adam (gfdt, dsm) (parallel branches)",
         ],
     },
+    "rl-ddpg-short": {
+        "abbreviation": "RL-DDPG-SHORT",
+        "Method": "Reinforcement Learning (DDPG, short RL step)",
+        "parallelism": "serial (short window)",
+        "update_type": "gradient (score)",
+        "method_goal": "optimization",
+        "emulator_use": "none",
+        "aliases": ["rl-ddpg-short"],
+    },
+    "rl-ddpg": {
+        "abbreviation": "RL-DDPG",
+        "Method": "Reinforcement Learning (DDPG)",
+        "parallelism": "serial",
+        "update_type": "gradient (score)",
+        "method_goal": "optimization",
+        "emulator_use": "none",
+        "aliases": ["rl-ddpg", "rl-ddpg-long"],
+    },
+    "rl-td3-short": {
+        "abbreviation": "RL-TD3-SHORT",
+        "Method": "Reinforcement Learning (TD3, short RL step)",
+        "parallelism": "serial (short window)",
+        "update_type": "gradient (score)",
+        "method_goal": "optimization",
+        "emulator_use": "none",
+        "aliases": ["rl-td3-short"],
+    },
+    "rl-td3": {
+        "abbreviation": "RL-TD3",
+        "Method": "Reinforcement Learning (TD3)",
+        "parallelism": "serial",
+        "update_type": "gradient (score)",
+        "method_goal": "optimization",
+        "emulator_use": "none",
+        "aliases": ["rl-td3", "rl-td3-long"],
+    },
 }
 
 

src/data_store.py

@@ -39,6 +39,10 @@ DATASET_FILES = {
         "opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l63_2026-08-12.nc",
         "opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l63_2026-09-11.nc",
         "opt_dsm_score_adam/leaderboard_sbadam_dsm_parallel_l63_2026-09-14.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-short_l63_2026-10-04.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-long_l63_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-short_l63_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-long_l63_2026-10-04.nc",
     ],
     "L96": [
         "teki_results/leaderboard_teki_l96_const-force_2026-09-10.nc",
@@ -51,6 +55,10 @@ DATASET_FILES = {
         "opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l96_const-force_2026-08-12.nc",
         "opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l96_const-force_2026-09-11.nc",
         "opt_dsm_score_adam/leaderboard_sbadam_dsm_parallel_l96_const-force_2026-09-14.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-short_l96_const-force_2026-10-04.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-long_l96_const-force_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-short_l96_const-force_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-long_l96_const-force_2026-10-04.nc",
     ],
     "L96_NN_FORCING": [
         "teki_results/leaderboard_teki_l96_flux-force_2026-09-10.nc",
@@ -61,6 +69,10 @@ DATASET_FILES = {
         "opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l96_flux-force_2026-08-12.nc",
         "opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l96_flux-force_2026-09-11.nc",
         "opt_dsm_score_adam/leaderboard_sbadam_dsm_parallel_l96_flux-force_2026-09-14.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-short_l96_flux-force_2026-10-04.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-long_l96_flux-force_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-short_l96_flux-force_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-long_l96_flux-force_2026-10-04.nc",
     ],
     "L96_SPATIAL_FORCING": [
         "teki_results/leaderboard_teki_l96_vec-force_2026-09-10.nc",
@@ -73,6 +85,10 @@ DATASET_FILES = {
         "opt_dsm_score_lm/leaderboard_sblm_dsm_parallel_l96_vec-force_2026-08-12.nc",
         "opt_kgmm_score_adam/leaderboard_sbadam_kgmm_parallel_l96_vec-force_2026-09-11.nc",
         "opt_dsm_score_adam/leaderboard_sbadam_dsm_parallel_l96_vec-force_2026-09-14.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-short_l96_vec-force_2026-10-04.nc",
+        "rl-ddpg/leaderboard_rl-ddpg-long_l96_vec-force_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-short_l96_vec-force_2026-10-04.nc",
+        "rl-td3/leaderboard_rl-td3-long_l96_vec-force_2026-10-04.nc",
     ],
 }
 

src/pages/MethodDetails.py

@@ -450,6 +450,178 @@ method_meta = {
         ],
         "notes": "**Inherits both LM-DSM's slow DSM-training cost and Adam's local-minimum susceptibility** — training the DSM score network is the binding wall-clock cost (as with LM-DSM), and Adam's lack of a trust region or ensemble diversity leaves it as prone to getting stuck in a bad basin as ADAM-KGMM.",
     },
+    "RL-DDPG-SHORT": {
+        "citation": "Lillicrap et al., ICLR, 2016",
+        "url": "https://arxiv.org/abs/1509.02971",
+        "source": "`opt_experiments/ReinforcementLearning` — DDPG actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `UPDATE_FREQUENCY=short`.",
+        "math": (
+            "The unknown forcing parameterization is the **policy** and the whitened statistics misfit "
+            "`Φ_k=‖y-G(x_{T_k})‖²_Γ` defines the **reward** `r_k=Φ_{k-1}-Φ_k`. A small critic MLP `q(x,F)` is "
+            "trained on replay-buffer Bellman targets `-(1-γ)Φ_k+γ·q(x_{k+1},π(x_{k+1}))` (the `Φ_{k-1}` "
+            "offset cancels exactly), and the policy parameters are ascended along `∇_F q` averaged over "
+            "buffer states with Adam. No Jacobian of the simulator is ever formed — the gradient comes "
+            "from differentiating the learned critic. **The prior enters in three ways, but not as a "
+            "penalty:** the initial policy is a random draw from the prior, the policy is stored in "
+            "prior-whitened coordinates `F=μ+Lθ̃` (`L` = Cholesky factor of the prior covariance) so the "
+            "Adam step and the Gaussian exploration noise are measured in prior standard deviations "
+            "(following the prior's correlations), and the critic sees the whitened action. The reward "
+            "is the misfit only. In the *short* variant one RL step advances a "
+            "sliding statistics window by 1/10 of its length, so there are ~10 RL updates per window of "
+            "simulated time, each costing a fraction of a forward evaluation."
+        ),
+        "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q_\vartheta(x,F)\right]_{F=F_\theta}\right)",
+        "guardrails": (
+            "`γ=0.5<1` is essential: for a state-independent policy at `γ=1` the reward telescopes and "
+            "`∇_F Q` vanishes, so `γ=1` reproduces the no-learning control. A Polyak-averaged target critic "
+            "(`τ=0.05`) stabilizes the bootstrap; critic inputs/targets are standardized from warm-up data "
+            "and frozen; Gaussian exploration noise (0.1 prior std) is added to the applied action, with a "
+            "warm-up phase before the first update. The statistic is computed over a trailing window of "
+            "the full length `W` (not the short step) so it is not biased by a shortened estimation "
+            "window. Cost is counted as simulation only (1 initial full evaluation + stride/T per step); "
+            "per-cell budget is capped at 10000 units."
+        ),
+        "params": [
+            {"Parameter": "γ (discount)", "Value": "0.5"},
+            {"Parameter": "Polyak rate", "Value": "0.05"},
+            {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
+            {"Parameter": "batch size / updates per step", "Value": "64 / 1"},
+            {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "10 / 50 / 5000"},
+            {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
+            {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
+        ],
+        "notes": (
+            "**Cost advantage is partly structural** — the continuing RL trajectory pays spin-up once rather "
+            "than every step (for L63 `T_start/T=0.75`), and the convergence test runs at every stride on "
+            "overlapping windows, so scores are not directly comparable to methods that re-pay spin-up each "
+            "evaluation. **Not tuned per case.** Converges on every seed on "
+            "L63 and L96-const, but the higher-dimensional forcings are the hard case for a learned "
+            "critic: on L96 flux-force and vec-force it reaches the targets on only a few of 30 seeds "
+            "(flux: 1 at target 1.1, 5 at 1.2; vec: 2 at 1.1, 4 at 1.2) and on none at 1.0 within the budget."
+        ),
+    },
+    "RL-DDPG": {
+        "citation": "Lillicrap et al., ICLR, 2016",
+        "url": "https://arxiv.org/abs/1509.02971",
+        "source": "`opt_experiments/ReinforcementLearning` — DDPG actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `UPDATE_FREQUENCY=long` (leaderboard name `rl-ddpg-long`).",
+        "math": (
+            "Same DDPG actor-critic scheme as RL-DDPG-SHORT (see that entry's update equation): the unknown "
+            "forcing parameterization is the **policy**, the decrease in the whitened statistics misfit "
+            "`Φ_k` is the **reward**, and the policy parameters are ascended along `∇_F q` of a small learned "
+            "critic. The prior enters as in RL-DDPG-SHORT: a random prior draw as the initial policy, "
+            "prior-whitened policy coordinates (step and exploration noise in prior standard deviations), "
+            "and no prior penalty in the reward. Here one RL step is the whole statistics window — exactly one standard forward "
+            "evaluation (spin-up + window), the same statistical window as every other method on the "
+            "leaderboard — so there is one RL update per forward evaluation and each step costs 1 unit."
+        ),
+        "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q_\vartheta(x,F)\right]_{F=F_\theta}\right)",
+        "guardrails": (
+            "Same `γ=0.5<1` (at `γ=1` the critic has no `F`-dependence for a state-independent policy), "
+            "Polyak target critic (`τ=0.05`), frozen input/target standardization, and Gaussian exploration "
+            "noise (0.1 prior std) as RL-DDPG-SHORT. The statistic `G_k` is exactly the standard forward map "
+            "used by every other method, and the spin-up is paid on every step; per-cell budget is capped "
+            "at 10000 units."
+        ),
+        "params": [
+            {"Parameter": "γ (discount)", "Value": "0.5"},
+            {"Parameter": "Polyak rate", "Value": "0.05"},
+            {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
+            {"Parameter": "batch size / updates per step", "Value": "64 / 1"},
+            {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "1 / 20 / 2000"},
+            {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
+            {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
+        ],
+        "notes": (
+            "**One RL update per forward evaluation, so far more expensive than RL-DDPG-SHORT** — the short "
+            "variant makes ~10× more updates per unit of cost and pays spin-up only once. **Not tuned per "
+            "case.** Converges reliably on L63 and L96-const, but never "
+            "reached the targets on L96 flux-force or vec-force within the budget."
+        ),
+    },
+    "RL-TD3-SHORT": {
+        "citation": "Fujimoto, van Hoof & Meger, ICML, 2018",
+        "url": "https://arxiv.org/abs/1802.09477",
+        "source": "`opt_experiments/ReinforcementLearning` — TD3 actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `ALGORITHM=td3`, `UPDATE_FREQUENCY=short`.",
+        "math": (
+            "Same actor-critic formulation as RL-DDPG-SHORT (see that entry): the unknown forcing "
+            "parameterization is the **policy**, the decrease in the whitened statistics misfit is the "
+            "**reward**, and the policy parameters are ascended along `∇_F q` of a learned critic. TD3 "
+            "replaces the single critic with **twin critics** whose minimum forms the Bellman target "
+            "`-(1-γ)Φ_k+γ·min_i q_i(x_{k+1},π(x_{k+1}))`, and the actor ascends critic 1 only. The prior "
+            "enters as in RL-DDPG-SHORT (random prior draw as the initial policy; prior-whitened policy "
+            "coordinates for the Adam step, exploration and target-smoothing noise; no prior penalty in "
+            "the reward). In the *short* variant one RL step advances a sliding statistics window by 1/10 of its length, so "
+            "there are ~10 RL updates per window of simulated time, each costing a fraction of a forward "
+            "evaluation."
+        ),
+        "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q^{(1)}_\vartheta(x,F)\right]_{F=F_\theta}\right)",
+        "guardrails": (
+            "The three TD3 robustness heuristics are switched on relative to RL-DDPG: **twin critics** with a "
+            "min-target (reduces critic overestimation), **delayed policy updates** (the actor and the "
+            "Polyak target critics update once per 2 critic updates), and **target-policy smoothing** "
+            "(clipped Gaussian noise added to the target action, in prior-std units, since the actor is a "
+            "single vector). As for RL-DDPG there is no target actor. Same `γ=0.5<1`, Polyak rate, frozen "
+            "input/target standardization, 0.1 prior-std Gaussian exploration noise and trailing window of "
+            "full length `W` as RL-DDPG-SHORT; cost counts simulation only (1 initial evaluation + stride/T "
+            "per step), capped at 10000 units per cell."
+        ),
+        "params": [
+            {"Parameter": "γ (discount)", "Value": "0.5"},
+            {"Parameter": "Polyak rate", "Value": "0.05"},
+            {"Parameter": "critics / policy delay", "Value": "2 / 2"},
+            {"Parameter": "target smoothing noise (clip)", "Value": "0.2 (0.5), prior-std units"},
+            {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
+            {"Parameter": "batch size / updates per step", "Value": "64 / 1"},
+            {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "10 / 50 / 5000"},
+            {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
+            {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
+        ],
+        "notes": (
+            "**Same structural cost advantage as RL-DDPG-SHORT** — the continuing trajectory pays spin-up "
+            "once and the convergence test runs on overlapping windows, so scores are not directly "
+            "comparable to methods that re-pay spin-up every evaluation. **Not tuned per case.** "
+            "Converges on every seed on L63 and L96-const (slower than RL-DDPG-SHORT on L63 at targets "
+            "1.0–1.1, similar at 1.2); on the higher-dimensional forcings it is weaker than RL-DDPG-SHORT: "
+            "it never reaches the targets on L96 flux-force, and on vec-force only 1 of 30 seeds does "
+            "(target 1.2) within the budget."
+        ),
+    },
+    "RL-TD3": {
+        "citation": "Fujimoto, van Hoof & Meger, ICML, 2018",
+        "url": "https://arxiv.org/abs/1802.09477",
+        "source": "`opt_experiments/ReinforcementLearning` — TD3 actor-critic (`ddpg_core.jl`, `rl_env.jl`, `rl_run.jl`), `ALGORITHM=td3`, `UPDATE_FREQUENCY=long` (leaderboard name `rl-td3-long`).",
+        "math": (
+            "Same TD3 actor-critic scheme as RL-TD3-SHORT (twin critics with a min-target; the actor "
+            "ascends critic 1 only; see that entry and RL-DDPG-SHORT for the formulation, including how "
+            "the prior sets the initial policy and the whitened coordinates). Here one RL step "
+            "is the whole statistics window — exactly one standard forward evaluation, the same statistical "
+            "window as every other method on the leaderboard — so there is one RL update per forward "
+            "evaluation and each step costs 1 unit."
+        ),
+        "math_eq": r"\theta \leftarrow \theta + \mathrm{Adam}\!\left(\mathbb{E}_{x}\left[\nabla_F\, q^{(1)}_\vartheta(x,F)\right]_{F=F_\theta}\right)",
+        "guardrails": (
+            "Same TD3 heuristics as RL-TD3-SHORT: twin critics with a min-target, delayed policy/target "
+            "updates (every 2 critic updates), and clipped target-policy smoothing noise. Same `γ=0.5<1`, "
+            "Polyak rate, frozen standardization and 0.1 prior-std exploration noise; the spin-up is paid "
+            "on every step; per-cell budget is capped at 10000 units."
+        ),
+        "params": [
+            {"Parameter": "γ (discount)", "Value": "0.5"},
+            {"Parameter": "Polyak rate", "Value": "0.05"},
+            {"Parameter": "critics / policy delay", "Value": "2 / 2"},
+            {"Parameter": "target smoothing noise (clip)", "Value": "0.2 (0.5), prior-std units"},
+            {"Parameter": "critic_lr / actor_lr", "Value": "1e-3 / 0.02 (prior-std units)"},
+            {"Parameter": "batch size / updates per step", "Value": "64 / 1"},
+            {"Parameter": "steps per window / warm-up steps / buffer size", "Value": "1 / 20 / 2000"},
+            {"Parameter": "exploration noise / warm-up fraction", "Value": "0.1 / 0.5"},
+            {"Parameter": "critic hidden width", "Value": "64 (L63, L96 const) / 128 (vec, flux)"},
+        ],
+        "notes": (
+            "**One RL update per forward evaluation, so far more expensive than RL-TD3-SHORT.** **Not tuned "
+            "per case.** Converges on every seed on L63 and L96-const (slower than RL-DDPG "
+            "on L96-const, and on L63 faster at targets 1.0–1.1 but slower at 1.2), but never reached the "
+            "targets on L96 flux-force or vec-force within the budget."
+        ),
+    },
 }
 
 # KNOWN_METHODS is the source of truth for the abbreviation/full-name list,

src/streamlit_app.py

@@ -121,9 +121,10 @@ _taxonomy_df = (
 )
 
 _UPDATE_TYPE_ROW_COLOR = {
-    "kalman": "#d9f2e6",    # pastel green
-    "gradient": "#dbe9fa",  # pastel blue
-    "general": "#fde2c8",   # pastel orange
+    "gradient (Kalman)": "#d9f2e6",  # pastel green
+    "gradient (exact)": "#dbe9fa",   # pastel blue
+    "gradient (score)": "#e6dcf5",   # pastel purple
+    "general": "#fde2c8",            # pastel orange
 }
 
 
@@ -151,7 +152,7 @@ st.dataframe(
 )
 st.caption(
     "Note: Kalman methods are Bayesian in spirit too (they're approximate Gaussian "
-    "posterior updates) — update type is about mechanism (gradient vs. Kalman vs. "
+    "posterior updates) — update type is about mechanism (which kind of gradient — exact, Kalman or score — vs. "
     "general), not whether a method is 'Bayesian'."
 )