ML models: walk-forward-judged panel rank, plain-language ML disclosure in report and progress

analyst.py

@@ -63,7 +63,7 @@ SEAT_RE = re.compile(r"^\s+(?:\w+ )?\((Neutral|Bear|Bull|PM|fallback)\)\s+(?:[-+
 # The framework's inputs for this company, shown while the storytellers work ("under the hood"). Only these
 # fields cross, each type-checked and bounded; values, ratings and storyteller views never do.
 FACT_FIELDS = {"industry": str, "country": str, "beta": float, "beta_basis": str, "erp": float, "erp_month": str,
-               "crp": float, "stage": str, "segments": list, "segment_build": bool, "consensus": bool}
+               "crp": float, "stage": str, "segments": list, "segment_build": bool, "consensus": bool, "ml_rank": str}
 
 
 def _facts(payload: str) -> dict:

engine/brief.py

@@ -42,7 +42,9 @@ def _quant_block(s):
                 else f"You may describe it only as '{pm.get('allowed_claim_strength')}'.")]
     lines.append(f"Cross-sectional panel rank: {pr.get('state')} ({pr.get('basis')}). "
                  + ("Not usable as evidence: its figures are withheld." if not pr.get("persona_visibility")
-                    else "Usable only as weak evidence."))
+                    else f"Usable only as {pr.get('allowed_claim_strength')}: a machine-learning model ranks the stock above "
+                         "[[ML_RANK_SHARE]] of the [[ML_RANK_UNIVERSE]] largest US companies (decile [[ML_RANK_DECILE]]) for "
+                         "12-month return relative to them. It is a ranking, not a price target."))
     return "\n".join(lines)
 
 

engine/catalog.py

@@ -131,6 +131,11 @@ CLAIMS = [
              ("forecast", "consensus")),
     ClaimDef("LIFECYCLE_FORWARD_GROWTH", "valuation.lifecycle.forward_revenue_growth", PCT_SIGNED, "forward revenue growth used for staging",
              ("trailing",)),
+    ClaimDef("ML_RANK_SHARE", "quant.panel_rank.figures.rank_share", PCT, "share of ranked companies below this one"),
+    ClaimDef("ML_RANK_DECILE", "quant.panel_rank.figures.decile", COUNT, "machine-learning rank decile"),
+    ClaimDef("ML_RANK_UNIVERSE", "quant.panel_rank.figures.n_universe", COUNT, "companies ranked"),
+    ClaimDef("ML_RANK_IC", "quant.panel_rank.reliability.rank_ic", RATIO, "walk-forward rank correlation"),
+    ClaimDef("ML_RANK_POSITIVE", "quant.panel_rank.reliability.ic_positive_share", PCT, "share of test months the rank was right-ordered"),
     ClaimDef("DMD_INDUSTRY", "valuation.damodaran_inputs.industry", TEXT, "industry (Damodaran classification)"),
     ClaimDef("DMD_UNLEVERED_BETA", "valuation.damodaran_inputs.unlevered_beta", RATIO, "industry unlevered beta (cash-corrected)"),
     ClaimDef("DMD_BETA", "valuation.damodaran_inputs.beta", RATIO, "bottom-up beta (re-levered at the company's D/E)"),

engine/policy.py

@@ -400,6 +400,28 @@ def panel_reliability(meta=None):
     if ic is None or sharpe is None:
         return reliability_record("UNAVAILABLE", basis="the panel snapshot carries no walk-forward validation metadata",
                                   extra={"model": "cross-sectional panel-ML rank"})
-    state = "VALIDATED" if ic >= 0.10 and sharpe >= 0.8 and n >= 24 else ("LIMITED" if ic >= 0.02 and n >= 12 else "UNRELIABLE")
-    return reliability_record(state, basis=f"walk-forward cross-sectional rank-IC {ic:.2f}, decile Sharpe {sharpe:.1f} over {n} periods",
-                              extra={"model": "cross-sectional panel-ML rank", "rank_ic": ic, "decile_sharpe": sharpe, "n_periods": n})
+    # 12-month windows of consecutive monthly cross-sections overlap: significance is judged on the
+    # non-overlapping count, and the latest test block must not have failed (today's regime is what a
+    # live analysis relies on; every approach went negative in 2024-25 in the prediction backtest)
+    t, n_ind = _f(m.get("ic_t")), _f(m.get("n_independent")) or n / 12.0
+    folds = m.get("folds") or []
+    recent = _f((folds[-1] if folds else {}).get("rank_ic"))
+    recent_ok = recent is not None and recent >= 0
+    if ic >= 0.10 and sharpe >= 0.8 and (t or 0) >= 2.0 and n_ind >= 8 and recent is not None and recent > 0:
+        state = "VALIDATED"
+    elif ic >= 0.02 and (t or 0) >= 1.0 and n_ind >= 5 and recent_ok:
+        state = "LIMITED"
+    else:
+        state = "UNRELIABLE"
+    why = ("" if state != "UNRELIABLE" else
+           "; the latest test block was negative" if (recent is not None and recent < 0) else
+           "; too weak or too few independent periods")
+    basis = (f"walk-forward rank correlation {ic:+.3f} (t {t if t is not None else float('nan'):.1f} over {n_ind:.1f} independent years), "
+             f"positive in {(_f(m.get('ic_positive_share')) or 0) * 100:.0f}% of months, decile Sharpe {sharpe:.2f}"
+             + (f", latest test block {(folds[-1] or {}).get('test')} {recent:+.3f}" if recent is not None else "") + why)
+    return reliability_record(state, basis=basis,
+                              extra={"model": "cross-sectional panel-ML rank", "rank_ic": ic, "decile_sharpe": sharpe, "n_periods": n,
+                                     "ic_t": t, "n_independent": n_ind, "recent_block_ic": recent,
+                                     "recent_block": (folds[-1] if folds else {}).get("test"),
+                                     "ic_positive_share": _f(m.get("ic_positive_share")),
+                                     "failed_recently": bool(recent is not None and recent < 0)})

engine/predict_yearly.py

@@ -7582,7 +7582,8 @@ Examples:
                       "stage": _CRUCIBLE_STORY_STAGE,
                       "segments": [g.get("name") for g in (_sbp.get("segments") or [])] if _sbp.get("filed") else [],
                       "segment_build": bool(_sbp.get("segment_build_required")),
-                      "consensus": bool(_eec.get("current_year_revenue"))}
+                      "consensus": bool(_eec.get("current_year_revenue")),
+                      "ml_rank": (_panel_state(ticker)[0] or {}).get("state")}
             print("@@progress story-start " + json.dumps(_facts, default=str), flush=True)
             story_val_result = _sd.story_value(_ctx, _story_call, _story_call2, _story_call3, verbose=args.verbose)
             if not story_val_result:

engine/report_model.py

@@ -412,6 +412,7 @@ def build_user_report(s, cat, prose, limitations):
     exc = [x.get("name") for x in (s.blend.get("excluded_sources") or ()) if x.get("name")]
     if exc:
         B.append(_para("blend", r.t("Carrying no weight in this analysis: " + ", ".join(exc) + ".")))
+    B += _ml_paragraphs(s, r, cat)
     B.append(_para("blend", r.t("The headline is the share-weighted sum of these views, a price return that excludes dividends: "),
                    r.c("HEADLINE_RETURN"), r.t(".")))
 
@@ -435,6 +436,29 @@ def build_user_report(s, cat, prose, limitations):
     return UserReport(s.identity.ticker, s.identity.get("company_name"), s.identity.get("as_of"), s.version, B, claims, limitations)
 
 
+def _ml_paragraphs(s, r, cat):
+    """Machine learning, in plain words. Two models are checked on every run; each counts only as far
+    as its out-of-sample record allows. A model that is not admitted is described without its figures."""
+    pm = (s.quant.get("price_model") or {}).get("reliability") or {}
+    pr = (s.quant.get("panel_rank") or {}).get("reliability") or {}
+    out = []
+    if pr.get("persona_visibility") and "ML_RANK_SHARE" in cat:
+        out.append(_para("blend", r.t("Machine learning: a model trained on twelve years of data ranks the stock above "), r.c("ML_RANK_SHARE"),
+                         r.t(" of the "), r.c("ML_RANK_UNIVERSE"), r.t(" largest US companies (decile "), r.c("ML_RANK_DECILE"),
+                         r.t(") for 12-month return relative to them. In walk-forward tests its rank correlation was "), r.c("ML_RANK_IC"),
+                         r.t(", right-ordered in "), r.c("ML_RANK_POSITIVE"),
+                         r.t(" of months. The analysts may cite it as weak evidence; it does not enter the headline return.")))
+    elif pr.get("state") == "UNRELIABLE":
+        why = ("it failed in the most recent test period" if pr.get("failed_recently") else "its out-of-sample record is too weak")
+        out.append(_para("blend", r.t("Machine learning: a model that ranks each stock against the largest US companies was checked "
+                                      f"and not used, because {why}.")))
+    if pm.get("state") in ("UNRELIABLE", "UNAVAILABLE"):
+        out.append(_para("blend", r.t("A single-stock machine-learning price model was also checked and not used: "
+                                      + ("its out-of-sample direction calls were no better than chance."
+                                         if pm.get("state") == "UNRELIABLE" else "there was not enough history to test it."))))
+    return out
+
+
 # ═══════════════════════════════════════════════════════════════════════════════════════
 # serialisation: both formats walk the same blocks
 # ═══════════════════════════════════════════════════════════════════════════════════════

engine/state_builder.py

@@ -401,8 +401,10 @@ def build_forecast_stage(state, *, pred_fund, fair_val, story, engine_run, engin
                                                 "rank_ic": _f((ml or {}).get("backtest_rank_ic")),
                                                 "mae": _f((ml or {}).get("backtest_mae")),
                                                 "n_oos_eff": _f((ml or {}).get("n_oos_effective"))}}
+    _pfig = (dict(panel_row, rank_share=(_f(panel_row.get("xs_rank_pct")) or 0.0) / 100.0)
+             if (panel_row and (panel_rel or {}).get("persona_visibility")) else None)
     s.quant.panel_rank = {"reliability": dict(panel_rel or {}),
-                          "figures": (dict(panel_row) if (panel_row and (panel_rel or {}).get("persona_visibility")) else None),
+                          "figures": _pfig,
                           "diagnostics_only": dict(panel_row) if panel_row else None}
 
     # macro, technical, sentiment

engine/tests/test_pipeline.py

@@ -161,7 +161,44 @@ def test_persona_brief_carries_tokens_not_quant_figures():
 
 def test_panel_reliability_requires_metadata():
     assert pol.panel_reliability(None)["state"] == "UNAVAILABLE"
-    assert pol.panel_reliability({"rank_ic": 0.05, "decile_sharpe": 0.6, "n_periods": 30})["state"] == "LIMITED"
+    # a record without a significance test or fold history cannot be admitted
+    assert pol.panel_reliability({"rank_ic": 0.05, "decile_sharpe": 0.6, "n_periods": 30})["state"] == "UNRELIABLE"
+
+
+def test_panel_reliability_judges_the_walk_forward_record():
+    """The shipped model's real record (2026-09-23 build): pooled rank IC +0.038, t 1.4 over 10.5 independent
+    years, but the latest block (2024-2025) was negative, so it is not admitted for a live analysis."""
+    real = {"rank_ic": 0.0383, "ic_t": 1.39, "ic_positive_share": 0.698, "decile_sharpe": 0.48, "n_periods": 126,
+            "n_independent": 10.5, "folds": [{"test": "2016-2017", "rank_ic": 0.1007}, {"test": "2022-2023", "rank_ic": 0.0393},
+                                              {"test": "2024-2025", "rank_ic": -0.0555}]}
+    r = pol.panel_reliability(real)
+    assert r["state"] == "UNRELIABLE" and r["failed_recently"] and "latest test block 2024-2025 -0.056" in r["basis"]
+    ok = dict(real, folds=real["folds"][:-1] + [{"test": "2024-2025", "rank_ic": 0.012}])
+    assert pol.panel_reliability(ok)["state"] == "LIMITED"
+    strong = dict(ok, rank_ic=0.12, ic_t=2.4, decile_sharpe=0.9)
+    assert pol.panel_reliability(strong)["state"] == "VALIDATED"
+    assert pol.panel_reliability(dict(ok, ic_t=0.7))["state"] == "UNRELIABLE"          # not significant
+
+
+def test_machine_learning_is_described_in_plain_words(good):
+    """Admitted: the rank and its record as audited claims. Not admitted: one sentence, no figures."""
+    import policy as pol2
+    d = F.all_data()
+    run, payload, failed = F.run_pipeline(data=d)
+    md = payload["markdown"]
+    assert failed is None and "single-stock machine-learning price model was also checked and not used" in md
+    rel = pol2.panel_reliability({"rank_ic": 0.05, "ic_t": 1.6, "ic_positive_share": 0.7, "decile_sharpe": 0.6, "n_periods": 120,
+                                  "n_independent": 10, "folds": [{"test": "2024-2025", "rank_ic": 0.02}]})
+    row = {"decile": 8, "xs_rank_pct": 74.0, "pred_excess_12m": 0.03, "as_of": "2026-09-18", "n_universe": 936}
+    run, payload, failed = F.run_pipeline(data=d, forecast_over={"panel_rel": rel, "panel_row": row})
+    md = payload["markdown"]
+    assert failed is None, failed
+    assert "ranks the stock above 74.0% of the 936 largest US companies (decile 8)" in md and "rank correlation was 0.05" in md
+    bad = pol2.panel_reliability({"rank_ic": 0.04, "ic_t": 1.4, "decile_sharpe": 0.5, "n_periods": 126, "n_independent": 10.5,
+                                  "folds": [{"test": "2024-2025", "rank_ic": -0.05}]})
+    run, payload, failed = F.run_pipeline(data=d, forecast_over={"panel_rel": bad, "panel_row": row})
+    assert failed is None and "checked and not used, because it failed in the most recent test period" in payload["markdown"]
+    assert "74.0%" not in payload["markdown"]                      # an excluded model's figures never appear
 
 
 # ── storytellers ─────────────────────────────────────────────────────────────────────

static/app.js

@@ -74,6 +74,8 @@ function valuationHood(f) {
     ? `${f.segments.join(", ")} have different economics, so each storyteller must value them one by one.`
     : `${f.segments.join(", ")} are valued together; their economics are close enough.`]);
   if (f.consensus) it.push(["Near term", "The first two years lean on analysts' revenue estimates; after that, growth fades toward the economy's."]);
+  if (f.ml_rank === "LIMITED" || f.ml_rank === "VALIDATED") it.push(["Machine learning", "A model trained on twelve years of data ranks the stock against the largest US companies. Its test record makes it supporting evidence for the analysts; it does not set the rating."]);
+  else if (f.ml_rank === "UNRELIABLE") it.push(["Machine learning", "A model that ranks the stock against the largest US companies was checked and switched off for now: its recent test record is too weak to rely on."]);
   it.push(["Storytellers", "Three different AI models are each writing a story at the same time: market size, share, margins and reinvestment."]);
   it.push(["Cash flows", "Each story becomes ten years of cash flow, discounted at today's rate for five years and then at a rate drifting toward a mature company's."]);
   it.push(["The value", "The middle of the three values is adopted, so no single storyteller sets it."]);

static/index.html

@@ -5,7 +5,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
 <meta name="theme-color" content="#0c0a0d">
 <title>Crucible</title>
-<link rel="stylesheet" href="/static/app.css?v=20260923x">
+<link rel="stylesheet" href="/static/app.css?v=20260923y">
 </head>
 <body>
 <header>
@@ -89,6 +89,6 @@
   <span class="mini-fill" id="mini-fill"></span><span class="mini-text" id="mini-text">Starting…</span>
 </button>
 
-<script src="/static/app.js?v=20260923x"></script>
+<script src="/static/app.js?v=20260923y"></script>
 </body>
 </html>