fix: hint guardrails, coached runs stay off the dataset, journal accounting fixes

README.md

@@ -189,8 +189,9 @@ Beam is the agent, and each Beam call is one step in a loop:
 2. **Plan and act.** Beam calls `act` with 1 to 8 moves. If something surprising happens partway
    (a new game message, lost HP, the run ending), the rest of the plan is dropped and Beam sees the
    new state before deciding again. It can call `write_notes` to keep working memory for the run.
-3. **Reflect.** When the run ends, Beam reads its notes and recent log and writes one to three
-   lessons for a new layout of the same level. These go into the journal.
+3. **Reflect.** When the run ends, every lesson Beam read is credited with the win or loss. Beam
+   then sees the journal with those records, plus its notes and recent log, and rewrites it for a
+   new layout of the same level (see Memory and self-improvement).
 
 Beam's reasoning comes back separately from its tool calls, so the page shows what it thought next
 to what it did. Beam is also the teacher: every call in a winning episode becomes a training
@@ -270,7 +271,8 @@ modal run scripts/modal_train.py::eval_entry --model Qwen/Qwen3-1.7B --tag base-
 ```
 
 The benchmark enforces a token cap across all parallel containers and can resume a stopped run
-without replaying finished seeds. `ChatClient` takes any OpenAI-compatible endpoint, so other
+without replaying finished seeds. `python scripts/compare_journal.py` rebuilds the memory
+comparison table in Results from the dataset. `ChatClient` takes any OpenAI-compatible endpoint, so other
 models can be run through the same harness.
 
 ## Limitations

app.py

@@ -27,6 +27,7 @@ from pydantic import BaseModel, Field
 
 from lantern.agent import EpisodeConfig, run_episode
 from lantern.coach import Hint, HintQueue
+from lantern.guard import check_hint
 from lantern.env import TASKS, DungeonEnv
 from lantern.journal import Journal
 from lantern.llm import ChatClient, OutOfCredits
@@ -104,8 +105,11 @@ class Run:
             env = DungeonEnv(TASKS[self.task_id], seed=self.seed)
             lessons = journal.lessons(self.task_id)
             # Journal v2: Beam curates the journal from each lesson's win/loss record.
+            # With "Use journal" off Beam never sees the lessons, so the run must neither credit
+            # them with its outcome nor rewrite the journal (same as the benchmark's "off").
             config = EpisodeConfig(use_lessons=self.use_lessons, max_llm_calls=MAX_CALLS_PER_RUN,
-                                   curate=True, lesson_records=journal.records(self.task_id))
+                                   reflect=self.use_lessons, curate=True,
+                                   lesson_records=journal.records(self.task_id) if self.use_lessons else None)
             for event in run_episode(env, ChatClient(), lessons, config, self.stop, self.hints):
                 if self.stop.is_set():
                     break
@@ -140,11 +144,13 @@ class Run:
         """Mark done and save; callers hold the lock."""
         self.done = True
         end = next((e for e in reversed(self.events) if e["type"] == "end"), {})
+        # Coached runs hold text typed by visitors (also quoted in Beam's reasoning), so they
+        # stay on the Space and never go to the public dataset.
         store.save({"id": self.id, "task": self.task_id, "seed": self.seed, "source": "space",
                     "use_lessons": self.use_lessons, "finished_at": time.time(),
                     "outcome": end.get("outcome", "error"), "steps": end.get("steps", 0),
                     "coached": self.coached,
-                    "events": list(self.events)})
+                    "events": list(self.events)}, upload=not self.coached)
 
 
 runs: dict[str, Run] = {}
@@ -301,6 +307,9 @@ def send_hint(run_id: str, req: HintRequest, request: Request) -> dict:
     text = clean_hint(req.text)
     if not text:
         raise HTTPException(400, "Type a hint first.")
+    problem = check_hint(text)   # PII, health information, profanity; the text itself is never logged
+    if problem:
+        raise HTTPException(400, f"That hint looks like it contains {problem}. Please keep hints about the game.")
     run.add_hint(Hint(text, req.seen_step, req.seen_x, req.seen_y))
     return {"queued": text}
 

lantern/agent.py

@@ -276,7 +276,9 @@ def run_episode(env: Env, client: ChatClient, lessons: list[str] | None = None,
 
     new_lessons: list[str] = []
     records: list[dict] | None = None
-    if config.reflect and outcome not in ("stopped",):
+    # A coached run's lessons never enter the shared journal (a viewer's text could ride along),
+    # so don't pay for a reflection that would be thrown away.
+    if config.reflect and outcome not in ("stopped",) and not advice_given:
         if config.curate:
             credited = journal.credit(config.lesson_records or [], outcome == "success")
             revised = curate_journal(client, outcome, steps, usage["calls"], config.max_llm_calls,
@@ -330,10 +332,15 @@ def curate_journal(client: ChatClient, outcome: str, steps: int, calls: int, max
     text = (reply.content or "").strip()
     start, end = text.find("{"), text.rfind("}")
     try:
-        revised = json.loads(text[start:end + 1]).get("journal", [])
+        parsed = json.loads(text[start:end + 1])
     except (json.JSONDecodeError, ValueError):
-        return [r["text"] for r in records]  # keep the journal as it was rather than lose it
-    return [str(x).strip()[:300] for x in revised if str(x).strip()]
+        parsed = None
+    revised = parsed.get("journal") if isinstance(parsed, dict) else None
+    if not isinstance(revised, list):
+        # Unreadable, the wrong shape, or a missing key: keep the journal as it was rather than
+        # erase it (or split a bare string into letters).
+        return [r["text"] for r in records]
+    return [str(x).strip()[:300] for x in revised if isinstance(x, str) and x.strip()]
 
 
 def reflect(client: ChatClient, outcome: str, steps: int, notes: str, log: list[str],

lantern/bench.py

@@ -26,7 +26,7 @@ def bench_seeds(n: int) -> list[int]:
 def play_series(task_id: str, seeds: list[int], use_lessons: bool, tag: str,
                 client: ChatClient, store: EpisodeStore, source: str = "bench",
                 stop: threading.Event | None = None,
-                initial_lessons: list[str] | None = None, curated: bool = False) -> list[dict]:
+                initial_lessons: list | None = None, curated: bool = False) -> list[dict]:
     """Play `seeds` in order on one level and save every episode; returns per-episode summaries.
 
     Each series starts with an empty journal; with `use_lessons`, lessons written after
@@ -35,8 +35,9 @@ def play_series(task_id: str, seeds: list[int], use_lessons: bool, tag: str,
     With `curated` (journal v2), lessons carry win/loss records and Beam rewrites the whole
     journal after each run instead of appending to it.
     """
-    lessons: list[str] = list(initial_lessons or [])  # non-empty when resuming a series
-    records = [journal.new_record(t) for t in lessons]
+    # Non-empty when resuming: lesson strings (v1) or records with win/loss counts (v2).
+    records = [r if isinstance(r, dict) else journal.new_record(r) for r in (initial_lessons or [])]
+    lessons = [r["text"] for r in records]
     summaries = []
     for seed in seeds:
         if stop is not None and stop.is_set():
@@ -117,21 +118,26 @@ class BudgetedClient:
         return reply
 
 
-def resume_state(records: list[dict], task_id: str, use_lessons: bool) -> tuple[set[int], list[str]]:
+def resume_state(records: list[dict], task_id: str, use_lessons: bool,
+                 curated: bool = False) -> tuple[set[int], list]:
     """Seeds already finished by a series, and the journal it had built, from saved episodes.
 
     Episodes cut off by the spending cap or by infrastructure errors count as unfinished
-    and are replayed.
+    and are replayed. A v1 journal comes back as lesson strings rebuilt with v1 rules; a
+    curated (v2) journal comes back as the last episode's records, win/loss counts intact.
     """
     done = sorted((r for r in records if r["task"] == task_id and r["use_lessons"] == use_lessons
                    and r["outcome"] not in NOT_RESULTS), key=lambda r: r["seed"])
-    lessons: list[str] = []
+    saved: list = []
     if use_lessons and done:
         last = done[-1]
         start = next((e for e in last["events"] if e["type"] == "start"), {})
         end = next((e for e in last["events"] if e["type"] == "end"), {})
-        lessons = merge_lessons(start.get("lessons", []), end.get("new_lessons", []), JOURNAL_LIMIT)
-    return {r["seed"] for r in done}, lessons
+        if curated:
+            saved = end.get("journal_records") or [journal.new_record(t) for t in start.get("lessons", [])]
+        else:
+            saved = merge_lessons(start.get("lessons", []), end.get("new_lessons", []), JOURNAL_LIMIT)
+    return {r["seed"] for r in done}, saved
 
 
 def summary_table(results: list[dict], task_ids: list[str], conditions: list[str]) -> str:

lantern/export.py

@@ -1,72 +1,75 @@
-"""Turn recorded episodes into supervised fine-tuning rows for a student model.
-
-Each Beam call becomes one chat example: the system prompt and observation Beam saw, and
-the tool calls it made. Only calls that executed something are kept, and by default only
-episodes Beam won, so the student imitates successful play.
-
-    python -m lantern.export data/ train.jsonl
-"""
-
-from __future__ import annotations
-
-import json
-import sys
-from pathlib import Path
-
-from .agent import PROMPT_VERSION
-from .store import EpisodeStore
-
-
-def tool_calls_for(executed: list[dict]) -> list[dict]:
-    """Rebuild OpenAI-style tool calls from the agent's executed-tool records."""
-    calls = []
-    for i, e in enumerate(executed):
-        if e["tool"] == "act":
-            args = {"actions": e["planned"]}
-        elif e["tool"] == "write_notes":
-            args = {"notes": e["notes"]}
-        else:
-            continue
-        calls.append({"id": f"call_{i}", "type": "function",
-                      "function": {"name": e["tool"], "arguments": json.dumps(args)}})
-    return calls
-
-
-def episode_rows(record: dict) -> list[dict]:
-    """SFT rows for one episode, or [] if it was recorded with a different prompt version."""
-    if record.get("coached"):
-        return []  # a person helped; the student should learn Beam's own play
-    start = next((e for e in record["events"] if e["type"] == "start"), None)
-    if not start or start.get("prompt_version") != PROMPT_VERSION:
-        return []  # older recordings used a different prompt; never mix them in
-    rows = []
-    for e in record["events"]:
-        if e["type"] != "turn" or not e["executed"]:
-            continue
-        rows.append({
-            "messages": [
-                {"role": "system", "content": start["system"]},
-                {"role": "user", "content": e["prompt"]},
-                {"role": "assistant", "content": "", "tool_calls": tool_calls_for(e["executed"])},
-            ],
-            "tools": start["tools"],
-            "task": record["task"], "seed": record.get("seed"), "episode": record["id"],
-        })
-    return rows
-
-
-def export(root: Path, out: Path, outcomes: tuple[str, ...] = ("success",)) -> int:
-    """Write SFT rows from every stored episode with a matching outcome; returns the row count."""
-    n = 0
-    with out.open("w", encoding="utf-8") as f:
-        for record in EpisodeStore(root).episodes():
-            if record.get("outcome") not in outcomes:
-                continue
-            for row in episode_rows(record):
-                f.write(json.dumps(row) + "\n")
-                n += 1
-    return n
-
-
-if __name__ == "__main__":
-    print(export(Path(sys.argv[1]), Path(sys.argv[2])), "rows")
+"""Turn recorded episodes into supervised fine-tuning rows for a student model.
+
+Each Beam call becomes one chat example: the system prompt and observation Beam saw, and
+the tool calls it made. Only calls that executed something are kept, and by default only
+episodes Beam won, so the student imitates successful play.
+
+    python -m lantern.export data/ train.jsonl
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+from .agent import PROMPT_VERSION
+from .llm import BEAM_MODEL
+from .store import EpisodeStore
+
+
+def tool_calls_for(executed: list[dict]) -> list[dict]:
+    """Rebuild OpenAI-style tool calls from the agent's executed-tool records."""
+    calls = []
+    for i, e in enumerate(executed):
+        if e["tool"] == "act":
+            args = {"actions": e["planned"]}
+        elif e["tool"] == "write_notes":
+            args = {"notes": e["notes"]}
+        else:
+            continue
+        calls.append({"id": f"call_{i}", "type": "function",
+                      "function": {"name": e["tool"], "arguments": json.dumps(args)}})
+    return calls
+
+
+def episode_rows(record: dict) -> list[dict]:
+    """SFT rows for one episode, or [] if it was recorded with a different prompt version."""
+    if record.get("coached"):
+        return []  # a person helped; the student should learn Beam's own play
+    start = next((e for e in record["events"] if e["type"] == "start"), None)
+    if not start or start.get("prompt_version") != PROMPT_VERSION:
+        return []  # older recordings used a different prompt; never mix them in
+    if start.get("model") != BEAM_MODEL:
+        return []  # only the teacher's moves; never the student's or the untrained model's
+    rows = []
+    for e in record["events"]:
+        if e["type"] != "turn" or not e["executed"]:
+            continue
+        rows.append({
+            "messages": [
+                {"role": "system", "content": start["system"]},
+                {"role": "user", "content": e["prompt"]},
+                {"role": "assistant", "content": "", "tool_calls": tool_calls_for(e["executed"])},
+            ],
+            "tools": start["tools"],
+            "task": record["task"], "seed": record.get("seed"), "episode": record["id"],
+        })
+    return rows
+
+
+def export(root: Path, out: Path, outcomes: tuple[str, ...] = ("success",)) -> int:
+    """Write SFT rows from every stored episode with a matching outcome; returns the row count."""
+    n = 0
+    with out.open("w", encoding="utf-8") as f:
+        for record in EpisodeStore(root).episodes():
+            if record.get("outcome") not in outcomes:
+                continue
+            for row in episode_rows(record):
+                f.write(json.dumps(row) + "\n")
+                n += 1
+    return n
+
+
+if __name__ == "__main__":
+    print(export(Path(sys.argv[1]), Path(sys.argv[2])), "rows")

lantern/guard.py

@@ -0,0 +1,78 @@
+"""Guardrails for the only free text strangers can send: coaching hints.
+
+Hints are shown to Beam and can be quoted in its reasoning, so they're checked for personal
+data (PII), health information (PHI) and profanity before they're accepted. Pattern checks
+catch the common cases, not everything; coached runs are also never uploaded to the public
+dataset, so anything that slips through stays on the Space.
+
+Nothing here logs hint text: the caller only gets back a reason category.
+"""
+
+from __future__ import annotations
+
+import re
+
+_PII = [
+    ("an email address", re.compile(r"[\w.+-]+@[\w-]+\.[a-z]{2,}", re.I)),
+    ("a web address", re.compile(r"(https?://|www\.)\S+", re.I)),
+    ("a Social Security number", re.compile(r"\b\d{3}-\d{2}-\d{4}\b")),
+    ("a phone number", re.compile(r"(?:\+?\d{1,3}[\s.-]?)?\(?\b\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}\b")),
+    ("an IP address", re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")),
+    ("a date", re.compile(r"\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b")),
+]
+_CARD = re.compile(r"\b(?:\d[ -]?){13,19}\b")
+# Game hints need small coordinates, never ID-like numbers: six digits in a row, or ten or more
+# split by spaces/dashes (e.g. a card number with a typo that fails the checksum).
+_LONG_NUMBER = re.compile(r"\d{6,}|\b(?:\d[ -]?){10,}\b")
+
+# Health-information terms that have no place in dungeon advice.
+_PHI = re.compile(r"\b(diagnos\w*|prescri\w*|medical record|mrn|patient|date of birth|dob|"
+                  r"health insurance|insurance (?:id|number)|medicaid|medicare|social security)\b", re.I)
+
+# General profanity, matched on whole words or word starts (so "fucking" matches but innocent
+# words that merely contain these letters don't). Slurs are deliberately not spelled out here.
+_PROFANE_WORDS = {"fuck", "shit", "cunt", "bitch", "bastard", "dick", "cock", "pussy", "asshole",
+                  "motherfucker", "bullshit", "wanker", "twat", "prick", "slut", "whore"}
+_PROFANE_STEMS = ("fuck", "shit", "motherf", "bullsh")
+_LEET = str.maketrans({"0": "o", "1": "i", "3": "e", "4": "a", "5": "s", "7": "t", "@": "a",
+                       "$": "s", "!": "i", "*": "u"})
+
+
+def _luhn(digits: str) -> bool:
+    total, double = 0, False
+    for d in reversed(digits):
+        n = int(d) * (2 if double else 1)
+        total += n - 9 if n > 9 else n
+        double = not double
+    return total % 10 == 0
+
+
+def _profane(text: str) -> bool:
+    lowered = text.lower()
+    # Undo spaced-out letters ("f u c k") and common symbol swaps before matching.
+    joined = re.sub(r"\b(\w)[\s.\-_]+(?=\w\b)", r"\1", lowered)
+    for variant in (lowered, joined):
+        words = re.findall(r"[\w@$!*]+", variant.translate(_LEET))
+        for word in words:
+            word = re.sub(r"(.)\1{2,}", r"\1", word)  # "fuuuck" -> "fuck"
+            if word in _PROFANE_WORDS or word.startswith(_PROFANE_STEMS):
+                return True
+    return False
+
+
+def check_hint(text: str) -> str | None:
+    """Why a hint can't be accepted, as a short phrase for the viewer; None if it's fine."""
+    for label, pattern in _PII:
+        if pattern.search(text):
+            return f"personal information ({label})"
+    for match in _CARD.finditer(text):
+        digits = re.sub(r"\D", "", match.group())
+        if 13 <= len(digits) <= 19 and _luhn(digits):
+            return "personal information (a card number)"
+    if _LONG_NUMBER.search(text):
+        return "personal information (a long ID-like number)"
+    if _PHI.search(text):
+        return "health information"
+    if _profane(text):
+        return "language we keep out of the demo"
+    return None

lantern/journal.py

@@ -51,12 +51,6 @@ class Journal:
         self._save()
         return self.lessons(task_id)
 
-    def add(self, task_id: str, new: list[str]) -> list[str]:
-        """Merge new lessons in the v1 way (append, newest wins), keeping existing records."""
-        known = {r["text"]: r for r in self._data.get(task_id, [])}
-        merged = merge_lessons(self.lessons(task_id), new, self.limit)
-        return self.set_records(task_id, [known.get(t, new_record(t)) for t in merged])
-
     def _save(self) -> None:
         self.path.parent.mkdir(parents=True, exist_ok=True)
         self.path.write_text(json.dumps(self._data, indent=2), "utf-8")
@@ -74,6 +68,14 @@ ACTION_WORDS = {
     "throw": "throw", "wield": "wield", "pick up": "pickup", "pickup": "pickup", "apply": "apply",
     "eat": "eat", "descend": "descend",
 }
+# "Search" is also everyday English ("search each corridor branch" just means explore), so it
+# only counts as the game command when phrased like one.
+COMMAND_PHRASES = {
+    "search": re.compile(r"\b(?:use|repeat|press)\s+(?:the\s+)?search\b"
+                         r"|\bsearch(?:es|ing)?\s+(?:for\s+)?(?:secret|hidden)\b"
+                         r"|\bsearch(?:es|ing)?\s+(?:the\s+|each\s+|every\s+)?(?:wall|walls|dead[- ]end)s?\b"
+                         r"|\bsearch(?:es|ing)?\s+(?:\w+\s+){0,3}(?:times|again|repeatedly|turns)\b"),
+}
 
 
 def new_record(text: str) -> dict:
@@ -90,7 +92,11 @@ def unavailable_action(text: str, legal_actions: list[str]) -> str | None:
     """The first action word in `text` that this level can't do, if any."""
     lowered = text.lower()
     for word, action in ACTION_WORDS.items():
-        if re.search(rf"\b{re.escape(word)}\b", lowered) and action not in legal_actions:
+        if action in legal_actions:
+            continue
+        phrase = COMMAND_PHRASES.get(word)
+        hit = phrase.search(lowered) if phrase else re.search(rf"\b{re.escape(word)}\b", lowered)
+        if hit:
             return word
     return None
 

lantern/store.py

@@ -41,13 +41,17 @@ class EpisodeStore:
         except Exception as exc:  # the app still works from local files
             log.warning("Could not pull episodes from %s: %s", self.repo_id, exc)
 
-    def save(self, record: dict, wait: bool = False) -> Path:
-        """Write one episode record locally, then upload it (in the background unless `wait`)."""
+    def save(self, record: dict, wait: bool = False, upload: bool = True) -> Path:
+        """Write one episode record locally, then upload it (in the background unless `wait`).
+
+        With `upload=False` the record stays local (e.g. runs containing visitors' text).
+        """
         rel = Path("episodes") / record["task"] / f"{record['id']}.json"
         path = self.root / rel
         path.parent.mkdir(parents=True, exist_ok=True)
         path.write_text(json.dumps(record), "utf-8")
-        self._upload([(path, rel.as_posix())], f"Add episode {record['id']} ({record['task']})", wait)
+        if upload:
+            self._upload([(path, rel.as_posix())], f"Add episode {record['id']} ({record['task']})", wait)
         return path
 
     def push_journal(self) -> None: