Merge pull request #35 from benjosaur/debug-modal-consistency

.env.example

@@ -4,6 +4,3 @@
 MODAL_JUDGE_URL=https://<your-workspace>--semantique-judge-judge-score.modal.run
 MODAL_KEY=wk-...
 MODAL_SECRET=ws-...
-
-# Offline alternative — no GPU, no network (stub judge):
-# JUDGE_FAKE=1

README.md

@@ -24,11 +24,11 @@ and uses up one hop. Reach `⏎` and a small open LLM
 judges what emotion your sentence expresses. Check off every target emotion
 on the list to collect them all.
 
-The judge uses **structured output via exact logprob filtering**: the prompt is
-your sentence followed by `is the same as`, and a single forward pass on a
-self-hosted open model scores how likely each candidate label is as the answer.
-The next-token distribution is masked to the board's labels and
-softmax-renormalized — constrained decoding made visible as the verdict card's
+The judge uses **structured output via exact logprob filtering**: a system prompt
+names the board's targets and asks for the one most similar to your sentence, and a
+single forward pass on a self-hosted open model scores how likely each candidate
+label is as the answer. The next-token distribution is masked to the board's labels
+and softmax-renormalized — constrained decoding made visible as the verdict card's
 probability bars.
 
 > Try it: for **happy**, `start → not → sad → ⏎` works.
@@ -74,8 +74,6 @@ cp .env.example .env           # fill in MODAL_JUDGE_URL + MODAL_KEY/MODAL_SECRE
 python app.py                  # http://127.0.0.1:7860
 ```
 
-No GPU/Modal handy? `JUDGE_FAKE=1 python app.py` runs an offline stub judge.
-
 Tests: `pytest -m "not slow"` (pure logic) · `pytest -m slow` (live eval against
 the Modal judge on every board-attainable sentence; skipped if `MODAL_JUDGE_URL`
 is unset).

judge.py

@@ -1,9 +1,8 @@
 """The judge: structured output via exact logprob filtering over a self-hosted LLM.
 
-A system message lists the level's still-needed targets, optionally followed by a
-few (sentence, label) examples as few-shot turns; the final user message is the
-assembled sentence followed by " is the same as". The model (on a Modal GPU — see
-modal_judge.py) scores each candidate label's whole token sequence in one forward
+A system message names the level's targets and asks for the one most similar to the
+player's message; the user message is the assembled sentence. The model (on a Modal
+GPU — see modal_judge.py) scores each candidate label's whole token sequence in one forward
 pass; we get the *exact* probability of each label, not just whatever lands in a
 hosted API's top-k. Those per-label logprobs are masked to the level's candidate
 labels and renormalized with a softmax — constrained decoding, enforced
@@ -11,7 +10,6 @@ client-side. The candidate set is what makes the answer an emotion on one board
 and an animal on another.
 
 `score_labels()` is the only seam to the model: it POSTs to the Modal endpoint.
-Set JUDGE_FAKE=1 for an offline stub (no GPU, no network).
 """
 
 import math
@@ -44,20 +42,17 @@ def assemble_sentence(words: list[str]) -> str:
     return out
 
 
-def build_messages(
-    sentence: str,
-    targets: list[str],
-    examples: tuple[tuple[str, str], ...] = (),
-) -> list[dict]:
-    """The judge prompt: a system message listing `targets`, then each (sentence,
-    label) in `examples` as a few-shot turn in the "<sentence> is the same as" ->
-    "<label>" shape, then the final "<sentence> is the same as"."""
-    messages = [{"role": "system", "content": f"The targets are: {', '.join(targets)}."}]
-    for ex_sentence, ex_label in examples:
-        messages.append({"role": "user", "content": f"{ex_sentence} is the same as"})
-        messages.append({"role": "assistant", "content": ex_label})
-    messages.append({"role": "user", "content": f"{sentence} is the same as"})
-    return messages
+def build_messages(sentence: str, targets: list[str]) -> list[dict]:
+    """The judge prompt: a system message naming `targets` and asking for the one
+    closest to the player's message, then the assembled sentence as the user turn."""
+    return [
+        {
+            "role": "system",
+            "content": f"The targets are: {', '.join(targets)}. "
+            "Output the one most similar to the user's message.",
+        },
+        {"role": "user", "content": sentence},
+    ]
 
 
 def renormalize(label_logprobs: dict[str, float]) -> dict[str, float]:
@@ -72,21 +67,20 @@ def score_labels(
     sentence: str,
     labels: list[str],
     targets: list[str],
-    examples: tuple[tuple[str, str], ...] = (),
 ) -> dict[str, float]:
     """One call to the Modal GPU endpoint -> exact per-label logprobs.
 
-    The prompt (built from `targets`/`examples`) is sent as chat messages; the
-    candidate `labels` are what the endpoint scores and constrains the answer to.
+    The prompt (built from `targets`) is sent as chat messages; the candidate
+    `labels` are what the endpoint scores and constrains the answer to.
     """
     if not MODAL_JUDGE_URL:
-        raise RuntimeError("MODAL_JUDGE_URL is not set (deploy modal_judge.py, or use JUDGE_FAKE=1)")
+        raise RuntimeError("MODAL_JUDGE_URL is not set (deploy modal_judge.py — see README)")
     headers = {}
     if MODAL_KEY and MODAL_SECRET:  # Modal proxy auth
         headers = {"Modal-Key": MODAL_KEY, "Modal-Secret": MODAL_SECRET}
     resp = requests.post(
         MODAL_JUDGE_URL,
-        json={"messages": build_messages(sentence, targets, examples), "labels": labels},
+        json={"messages": build_messages(sentence, targets), "labels": labels},
         headers=headers,
         timeout=REQUEST_TIMEOUT,
     )
@@ -103,7 +97,7 @@ def judge(payload: dict) -> dict:
     round-trip through the client.
     """
     words = payload["words"]
-    targets = payload["targets"]  # still-unchecked labels; any of them wins
+    remaining = payload["targets"]  # still-unchecked labels; only the win check (+ stub) use these
     level = get_level(payload["level_id"])
     labels = level.labels
     if not words:
@@ -116,14 +110,9 @@ def judge(payload: dict) -> dict:
         }
     sentence = assemble_sentence(words)
     try:
-        if os.environ.get("JUDGE_FAKE"):  # offline dev mode: no GPU, no network
-            # Favour the first still-needed target so a collect-them-all run
-            # is playable without a token.
-            fake = {label: -6.0 - i for i, label in enumerate(labels)}
-            fake[targets[0] if targets else labels[0]] = -0.3
-            probs = renormalize(fake)
-        else:
-            probs = renormalize(score_labels(sentence, labels, targets, level.examples))
+        # Condition on the full target set so the prompt is stable across a session;
+        # the win check below, not the prompt, is what tracks remaining targets.
+        probs = renormalize(score_labels(sentence, labels, level.targets))
     except Exception as e:
         return {"ok": False, "error": str(e)}
     winner = max(probs, key=probs.get)
@@ -132,5 +121,5 @@ def judge(payload: dict) -> dict:
         "sentence": sentence,
         "probs": probs,
         "winner": winner,
-        "verdict": "win" if winner in targets else "lose",
+        "verdict": "win" if winner in remaining else "lose",
     }

levels/_base.py

@@ -24,9 +24,6 @@ class Level:
     targets: list[str]  # labels to collect (a win checks one off)
     labels: list[str]  # full candidate set the judge scores among
     budget: int  # hops before the doodle falls off the board
-    # Few-shot (sentence, label) pairs prepended to the judge prompt to calibrate
-    # its read of this board. Judge-only — never shipped to the client (no spoilers).
-    examples: tuple[tuple[str, str], ...] = ()
     order: int = 100  # play order across boards (lower comes first)
     home: bool = False  # the board the app boots into
 

levels/animal.py

@@ -21,12 +21,6 @@ LEVEL = Level(
     start=(2, 2),  # the centre tile
     targets=["whale", "dolphin", "otter", "cat", "dog", "bear", "owl", "mouse", "lion", "cow"],
     labels=["whale", "dolphin", "otter", "cat", "dog", "bear", "owl", "mouse", "lion", "cow"],
-    examples=(  # few-shot calibration for the judge (board words hinting animals)
-        ("sea big", "whale"),
-        ("puppy", "dog"),
-        ("flying quiet", "owl"),
-        ("grass big", "cow"),
-    ),
     budget=12,
     order=1,
 )

levels/emotion.py

@@ -20,11 +20,6 @@ LEVEL = Level(
     start=(1, 1),  # row, col of the start tile
     targets=["happy", "betrayed", "surprised"],  # collect them all
     labels=["happy", "betrayed", "surprised"],
-    examples=(  # few-shot calibration for the judge (board words, all three labels)
-        ("great!", "happy"),
-        ("hurt", "betrayed"),
-        ("yikes", "surprised"),
-    ),
     budget=10,
     order=0,
     home=True,

modal_judge.py

@@ -167,8 +167,12 @@ def main(sentence: str = "not sad"):
     """Smoke test against the deployed/served class: `modal run modal_judge.py`."""
     labels = ["happy", "betrayed", "surprised"]
     messages = [
-        {"role": "system", "content": f"The targets are: {', '.join(labels)}."},
-        {"role": "user", "content": f"{sentence} is the same as"},
+        {
+            "role": "system",
+            "content": f"The targets are: {', '.join(labels)}. "
+            "Output the one most similar to the user's message.",
+        },
+        {"role": "user", "content": sentence},
     ]
     result = Judge().score_remote.remote(messages, labels)
     print(f"{sentence!r}")

scripts/check_modal.py

@@ -36,8 +36,8 @@ SENTENCES = ["not sad", "great!", "very great!", "hurt", "yikes", "great sad"]
 
 
 def call(sentence: str) -> dict:
-    # Mirror the in-game call: every label is a target, board examples calibrate.
-    messages = build_messages(sentence, LEVEL.labels, LEVEL.examples)
+    # Mirror the in-game call: every label is a target.
+    messages = build_messages(sentence, LEVEL.labels)
     resp = requests.post(
         URL,
         json={"messages": messages, "labels": LEVEL.labels, "debug": True},

tests/test_judge.py

@@ -46,13 +46,11 @@ def test_live_judge_on_board_sentences(sentence, target, should_win):
     from judge import score_labels
     from levels import get_level
 
-    # Mirror the in-game judge() call: every label is passed as a target (so the
-    # system prompt hints no single answer) and the board's own few-shot examples
-    # calibrate the read. This measures the discrimination players actually get,
-    # which is what makes labels like "betrayed"/"surprised" reachable at all.
+    # Mirror the in-game judge() call: every label is passed as a target, so the
+    # system prompt names them all and asks for the one closest to the sentence.
+    # This measures the discrimination players actually get, which is what makes
+    # labels like "betrayed"/"surprised" reachable at all.
     level = get_level("emotion")
-    probs = renormalize(
-        score_labels(sentence, level.labels, targets=level.labels, examples=level.examples)
-    )
+    probs = renormalize(score_labels(sentence, level.labels, targets=level.labels))
     winner = max(probs, key=probs.get)
     assert (winner == target) == should_win, f"{sentence!r} -> {probs}"