sync: from ael backend/space/

rerank.py

@@ -135,6 +135,11 @@ MIN_KEEP = int(os.environ.get("AEL_RERANK_MIN_KEEP", "2"))
 # bounds the work, and this is only a backstop against a pathologically long set.
 MAX_SCORED = int(os.environ.get("AEL_RERANK_MAX", "10"))
 
+# How much text the model is asked to judge. The model's context is 512 tokens;
+# a title plus a full snippet can exceed it and the model raises rather than
+# truncating, which used to surface as a silent per-query failure.
+PASSAGE_MAX_CHARS = int(os.environ.get("AEL_RERANK_PASSAGE_CHARS", "1200"))
+
 # Recent (query, scores) for threshold tuning. Bounded so a long-lived Space
 # cannot grow without limit.
 RECENT = deque(maxlen=int(os.environ.get("AEL_RERANK_LOG", "200")))
@@ -289,10 +294,23 @@ def _question(query):
 
 def _passage(result):
     """The text the model judges. Title carries most of the topical signal;
-    snippet is what we have of the page without fetching it."""
+    snippet is what we have of the page without fetching it.
+
+    Truncated to PASSAGE_MAX_CHARS. Two reasons, and only the first was
+    obvious: a snippet plus title can run past the model's context, and the
+    model raises rather than truncating — which surfaced as a per-query
+    `ok: false` with no reason, reproducible only for certain queries. The
+    second is latency: a document costs ~150ms on the Space's CPU and most of
+    that is attention over the input, so the cap pays for itself twice.
+    """
     title = str(result.get("title") or "").strip()
     snippet = str(result.get("snippet") or "").strip()
-    return (title + "\n" + snippet).strip()
+    text = (title + "\n" + snippet).strip()
+    if len(text) > PASSAGE_MAX_CHARS:
+        # Title first: it survives the cut, and it is the part carrying the
+        # topical signal.
+        text = title[:PASSAGE_MAX_CHARS] + "\n" + snippet[: max(0, PASSAGE_MAX_CHARS - len(title))]
+    return text
 
 
 def _stamp(out):
@@ -362,7 +380,12 @@ def rerank(query, results, budget_ms=800):
             p = float(res["answers"]["relevant"]["noul"])
         except Exception as e:  # noqa: BLE001
             print(f"[rerank] score failed for result {i}: {type(e).__name__}: {e}", flush=True)
-            return _stamp(dict(out, results=results))  # fail open, whole set untouched
+            # Fail open, whole set untouched — and say why. A scoring failure
+            # used to be the same three words as a reranker that was not
+            # installed, which is how a per-query failure hid for a day.
+            out["results"] = results
+            out["no_opinion"] = f"scoring failed: {type(e).__name__}: {e}"[:300]
+            return out
         scored.append((p, i, r))
         judged += 1
 
@@ -377,7 +400,11 @@ def rerank(query, results, budget_ms=800):
     out["judged"] = judged
 
     if not any(p is not None for p, _, _ in scored):
-        return _stamp(out)
+        # Every result was textless, so the model was never asked anything.
+        # That is a different thing from the model declining, and it is the
+        # kind of difference that is invisible until it is named.
+        out["no_opinion"] = "no result carried any text to judge"
+        return out
 
     # Highest score first. Unscored results keep their relative position at the
     # end rather than being dropped: we have no opinion about them.