Sync from GitHub via hub-sync

dashboard/src/playground/sessions/SleepPanel.tsx

@@ -63,6 +63,7 @@ function RunCard({ run }: { run: SleepRun }) {
               {' → '}
               {after?.recall ? `${after.recall.recalled}/${after.recall.n_probes}` : 'n/a'}
               {after?.recall && after.recall.n_paraphrase > 0 ? ` (paraphrase ${before?.recall?.recalled_paraphrase ?? 'n/a'} → ${after.recall.recalled_paraphrase}/${after.recall.n_paraphrase})` : ''}
+              {typeof after?.recall?.distinct_ratio === 'number' ? ` · distinct replies ${fmt((before?.recall?.distinct_ratio ?? 0) * 100, 2)}% → ${fmt(after.recall.distinct_ratio * 100, 2)}%` : ''}
             </dd>
           </div>
           <div>

dashboard/src/playground/types.ts

@@ -58,7 +58,7 @@ export interface SleepOptions {
 export interface SleepMeasurement {
   heldout_nll: { mean: number | null; median: number | null; tokens: number } | null;
   canary: { coherence: number | null; poison: number | null } | null;
-  recall: { n_probes: number; recalled: number; recalled_exact: number; n_paraphrase: number; recalled_paraphrase: number } | null;
+  recall: { n_probes: number; recalled: number; recalled_exact: number; n_paraphrase: number; recalled_paraphrase: number; distinct_ratio?: number | null } | null;
 }
 
 export interface SleepReport {

docs/research/2026-09-23-sleep-consolidation.md

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7382e93151efcde968f10c27143fe06346efa7c46a98097af4d638364ac479e1
-size 21477
+oid sha256:db7af338b954581acf44921040bedd9e736ea53a33b21844709e34f776641f2b
+size 23335

plastic/sleep/recall.py

@@ -78,6 +78,14 @@ class RecallReport:
     def count(self, *, variant: str | None = None, key: str = "contains") -> int:
         return sum(1 for r in self.results if (variant is None or r.variant == variant) and getattr(r, key))
 
+    def distinct_ratio(self) -> float | None:
+        """Share of distinct replies over all probes (normalized, first 12 words). A model that answers every
+        question with the same sentence has collapsed; a perplexity gate cannot see that, this can."""
+        if not self.results:
+            return None
+        keys = {" ".join(normalize(r.reply).split()[:12]) for r in self.results}
+        return len(keys) / len(self.results)
+
     def to_dict(self) -> dict[str, Any]:
         verbatim = [r for r in self.results if r.variant == "verbatim"]
         para = [r for r in self.results if r.variant == "paraphrase"]
@@ -87,6 +95,7 @@ class RecallReport:
             "recalled_exact": sum(r.exact for r in verbatim),
             "n_paraphrase": len(para),
             "recalled_paraphrase": sum(r.contains for r in para),
+            "distinct_ratio": self.distinct_ratio(),
             "results": [asdict(r) for r in self.results],
         }
 

plastic/sleep/ttt.py

@@ -67,6 +67,9 @@ class SleepConfig:
     distill_temperature: float = 1.0
     tolerance_nll: float = 0.05        # allowed rise in mean held-out assistant NLL (nats per token)
     tolerance_canary: dict[str, float] = field(default_factory=lambda: {"coherence": 0.1, "poison": 0.1})
+    # behavioral locality: the share of distinct probe replies may not fall below this when it was higher before.
+    # A run that lowers held-out NLL while every reply becomes the same sentence has collapsed, not learned.
+    tolerance_collapse: float = 0.5
     recall_max_new_tokens: int = 48
     seed: int = 0
     device: str = "cpu"
@@ -509,7 +512,8 @@ def sleep_ttt(
     }
     report["after"] = after
     log(f"[sleep] after:  heldout NLL {_fmt(after['heldout_nll'])}; recall {_fmt_recall(after['recall'])}")
-    gate = gate_from_measurements(before, after, tolerance_nll=cfg.tolerance_nll, tolerance_canary=cfg.tolerance_canary)
+    gate = gate_from_measurements(before, after, tolerance_nll=cfg.tolerance_nll, tolerance_canary=cfg.tolerance_canary,
+                                  tolerance_collapse=cfg.tolerance_collapse)
     report["gate"] = gate
     if before["recall"] and after["recall"]:
         report["recall_gain"] = {"recalled": after["recall"]["recalled"] - before["recall"]["recalled"],
@@ -552,9 +556,14 @@ def sleep_ttt(
     return report
 
 
-def gate_from_measurements(before: dict[str, Any], after: dict[str, Any], *, tolerance_nll: float, tolerance_canary: dict[str, float]) -> dict[str, Any]:
+def gate_from_measurements(before: dict[str, Any], after: dict[str, Any], *, tolerance_nll: float, tolerance_canary: dict[str, float],
+                           tolerance_collapse: float = 0.5) -> dict[str, Any]:
     """The locality gate. ``measured`` says whether any check could run; ``passed`` is None when nothing was
-    measured, so an unmeasured run is never reported as verified. A nonfinite measurement fails its check."""
+    measured, so an unmeasured run is never reported as verified. A nonfinite measurement fails its check.
+    With recall probes present, the share of distinct replies after sleep may not drop below
+    ``tolerance_collapse`` when it was at or above it before: a perplexity drop with collapsed replies is a
+    failure (observed on the step-100 checkpoint, 40 steps on all parameters: NLL fell, 6 of 15 answers
+    became the same sentence)."""
     gate: dict[str, Any] = {"passed": None, "measured": False, "checks": []}
 
     def check(name: str, value: float, limit: float, ok: bool) -> None:
@@ -570,6 +579,10 @@ def gate_from_measurements(before: dict[str, Any], after: dict[str, Any], *, tol
         lim_c, lim_p = tolerance_canary.get("coherence", 0.1), tolerance_canary.get("poison", 0.1)
         check("canary_coherence_rise", d_coh, lim_c, d_coh <= lim_c)
         check("canary_poison_drop", d_poi, lim_p, d_poi >= -lim_p)
+    rb, ra = (before.get("recall") or {}).get("distinct_ratio"), (after.get("recall") or {}).get("distinct_ratio")
+    if rb is not None and ra is not None:
+        ok = (ra >= tolerance_collapse) or (ra >= rb)  # a run may not push distinct replies below the floor unless they already were
+        check("reply_distinct_ratio", ra, tolerance_collapse, ok)
     gate["measured"] = bool(gate["checks"])
     gate["passed"] = all(c["passed"] for c in gate["checks"]) if gate["checks"] else None
     if not gate["measured"]:

source_snapshot.json

@@ -1,6 +1,6 @@
 {
   "github_repository": "DMontgomery40/plastic",
-  "github_commit": "93d524decf049ff32b7c462fb860e7c9ca319700",
-  "preserved_hf_revision": "62148b3901210d41a402c1c4f89c3811c5050a81",
+  "github_commit": "a11157bf3cf76829e61c5f16488f11009743a622",
+  "preserved_hf_revision": "72d3e2e7b1548cf6ab1f4594613a3a59f9b33a03",
   "repo_type": "space"
 }

tests/test_sleep_ttt.py

@@ -186,6 +186,9 @@ def test_recall_scoring_and_probe_loading(tmp_path):
     replies = {"What is my cat's name?": "Marlowe.", "Remind me what I call my cat.": "A cat.", "Where do I live?": "You live in Denver."}
     rep = run_probes(probes, lambda q: replies[q])
     d = rep.to_dict()
+    assert d["distinct_ratio"] == 1.0
+    collapsed = run_probes(probes, lambda q: "A pleasure to meet you, Marlowe. I am glad you asked.").to_dict()
+    assert collapsed["distinct_ratio"] == 1 / 3 and collapsed["recalled"] == 1  # every reply the same; only the cat probe "hits"
     assert (d["n_probes"], d["recalled"], d["recalled_exact"], d["n_paraphrase"], d["recalled_paraphrase"]) == (2, 2, 1, 1, 0)
     assert isinstance(rep, RecallReport) and rep.count(variant="paraphrase") == 0
     with pytest.raises(ValueError):
@@ -206,6 +209,15 @@ def test_gate_treats_nonfinite_measurements_as_failures():
     none = gate_from_measurements({"heldout_nll": None, "canary": None, "recall": None}, {"heldout_nll": None, "canary": None, "recall": None},
                                   tolerance_nll=0.05, tolerance_canary={})
     assert none["measured"] is False and none["passed"] is None and "exploratory" in none["note"]
+    # behavioral collapse: NLL improves but replies collapse onto one sentence -> the gate fails
+    collapsed = gate_from_measurements({"heldout_nll": {"mean": 1.68, "median": 0.9, "tokens": 800}, "canary": None, "recall": {"distinct_ratio": 1.0}},
+                                       {"heldout_nll": {"mean": 1.63, "median": 0.4, "tokens": 800}, "canary": None, "recall": {"distinct_ratio": 0.4}},
+                                       tolerance_nll=0.05, tolerance_canary={})
+    assert collapsed["passed"] is False and [c["name"] for c in collapsed["checks"] if not c["passed"]] == ["reply_distinct_ratio"]
+    # a model whose replies were already repetitive is not blamed for staying so, and improvement always passes
+    same = gate_from_measurements({"heldout_nll": None, "canary": None, "recall": {"distinct_ratio": 0.3}},
+                                  {"heldout_nll": None, "canary": None, "recall": {"distinct_ratio": 0.35}}, tolerance_nll=0.05, tolerance_canary={})
+    assert same["passed"] is True
 
 
 def test_sleep_config_validation():