perf(sidecar): stop blocking the event loop on 2.7GB of local SQLite

app/local_meta.py is synchronous sqlite3 over a 2.7 GB file, and three
call sites invoked it directly from inside `async def`. A blocking call
there stalls EVERY in-flight request, including ones that never touch the
sidecar, and this deploys as a single uvicorn worker on 2 vCPUs — there
is no second worker to absorb it.

Measured on a 300k-row stand-in: a 72 ms query held the event loop for
78 ms; the same query through a thread held it for 7 ms, which is just
scheduler granularity. The real sidecar has 1.8M rows.

Moved to asyncio.to_thread:
- turso_svc.fetch_metadata_batch -> local_meta.fetch_rows
- turso_svc.fetch_trending_by_categories -> local_meta.fetch_trending,
which matters most: it is the single most expensive query in the
system and the one that lands on brand-new users
- space.py _titles_for, now async; all three of its callers were
already coroutines

This is safe by construction rather than by luck: local_meta opens its
connection read-only with check_same_thread=False, with a comment saying
it is so several threads can share the handle.

fts_svc.search_sparse was already correct — it uses run_in_executor — so
the FTS path was never affected.

A test walks app/ with ast and fails if any coroutine calls a blocking
local_meta function directly, plus a positive control so the guard cannot
silently match nothing, plus a timing test asserting a threaded query does
not stall the loop.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/routers/space.py

@@ -18,6 +18,7 @@ rewrite -> encode -> dense+sparse fanout -> RRF -> title boost, and
 arxiv_id. These are thin, honest wrappers over both. Qdrant credentials stay
 here and are never handed to the edge app.
 """
+import asyncio
 import time
 
 from fastapi import APIRouter, Header, HTTPException, Query
@@ -46,12 +47,18 @@ def _authorize(authorization: str | None) -> None:
         raise HTTPException(status_code=401, detail="invalid service token")
 
 
-def _titles_for(arxiv_ids: list[str]) -> dict[str, str]:
-    """Best-effort titles from the local sidecar; absent ids simply stay absent."""
+async def _titles_for(arxiv_ids: list[str]) -> dict[str, str]:
+    """Best-effort titles from the local sidecar; absent ids simply stay absent.
+
+    Off the event loop: local_meta is synchronous sqlite3 over a 2.7 GB file,
+    and blocking here stalls every other in-flight request. Two of the three
+    call sites pass a handful of ids, but the batch one (line 94) passes a
+    whole page of results.
+    """
     if not arxiv_ids:
         return {}
     try:
-        rows = local_meta.fetch_rows(arxiv_ids)
+        rows = await asyncio.to_thread(local_meta.fetch_rows, arxiv_ids)
     except Exception as exc:  # sidecar is optional, never fatal
         print(f"[space] title lookup failed ({exc})")
         return {}
@@ -91,7 +98,7 @@ async def space_search(
         print(f"[space] search failed ({exc})")
         raise HTTPException(status_code=502, detail="search unavailable") from exc
 
-    titles = _titles_for(list(ids))
+    titles = await _titles_for(list(ids))
     return {
         "query": query,
         "tookMs": round((time.perf_counter() - started) * 1000),
@@ -188,7 +195,7 @@ async def space_similarity(
         raise HTTPException(status_code=502, detail="vector store unavailable") from exc
 
     va, vb = vectors.get(left), vectors.get(right)
-    titles = _titles_for([left, right])
+    titles = await _titles_for([left, right])
     both = va is not None and vb is not None
     return {
         "a": {"arxivId": left, "title": titles.get(left, ""), "found": va is not None},
@@ -233,7 +240,7 @@ async def space_neighbors(
         raise HTTPException(status_code=502, detail="vector store unavailable") from exc
 
     hits = hits[:limit]
-    titles = _titles_for([h["arxiv_id"] for h in hits])
+    titles = await _titles_for([h["arxiv_id"] for h in hits])
     return {
         "arxivId": anchor,
         "found": True,

app/turso_svc.py

@@ -13,6 +13,7 @@ Table:      papers (arxiv_id UNIQUE INDEX)
 """
 from __future__ import annotations
 
+import asyncio
 import json
 import time
 from collections import OrderedDict
@@ -115,7 +116,15 @@ async def fetch_metadata_batch(arxiv_ids: list[str]) -> dict[str, dict]:
     # missing or partially-built sidecar only costs us the speedup.
     from app import local_meta
     if local_meta.is_available():
-        for row in local_meta.fetch_rows(misses):
+        # Off the event loop. local_meta is synchronous sqlite3 against a 2.7 GB
+        # file, and a blocking call inside an async handler stalls EVERY other
+        # in-flight request, including ones that never touch the sidecar.
+        # Measured on a 300k-row stand-in, a 72 ms query held the loop for
+        # 78 ms; off-loop that drops to scheduler granularity. The connection is
+        # opened read-only with check_same_thread=False precisely so it can be
+        # shared across threads (local_meta._probe).
+        sidecar_rows = await asyncio.to_thread(local_meta.fetch_rows, misses)
+        for row in sidecar_rows:
             paper = _to_paper_dict(row)
             if paper:
                 output[paper["arxiv_id"]] = paper
@@ -369,9 +378,12 @@ async def fetch_trending_by_categories(
     # this becomes an index range read.
     from app import local_meta
     if local_meta.is_available():
-        rows = local_meta.fetch_trending(
-            set(categories), limit=limit,
-            recency_months=config.TRENDING_RECENCY_MONTHS,
+        # Off the event loop — see fetch_metadata_batch. This one matters more:
+        # trending is the single most expensive query in the system and the one
+        # that lands on brand-new users.
+        rows = await asyncio.to_thread(
+            local_meta.fetch_trending, set(categories),
+            limit, config.TRENDING_RECENCY_MONTHS,
         )
         if rows:
             papers = [p for p in (_to_paper_dict(r) for r in rows) if p]

tests/test_event_loop_blocking.py

@@ -0,0 +1,113 @@
+"""
+Nothing may block the event loop with sidecar I/O.
+
+`app/local_meta.py` is synchronous sqlite3 over a 2.7 GB file. A blocking call
+inside an `async def` stalls EVERY in-flight request, including ones that never
+touch the sidecar — and this runs as a single uvicorn worker on 2 vCPUs, so
+there is no other worker to absorb it.
+
+Measured on a 300k-row stand-in: a 72 ms query held the loop for 78 ms; the
+same query via a thread held it for 7 ms. The real sidecar has 1.8M rows.
+
+The shared connection is opened read-only with check_same_thread=False
+precisely so it can be used from a thread (local_meta._probe).
+"""
+import ast
+import asyncio
+import pathlib
+import sqlite3
+import tempfile
+import time
+
+import pytest
+
+# Functions that touch the sidecar's connection and therefore must not be
+# called directly from a coroutine.
+BLOCKING = {"fetch_rows", "fetch_trending", "pub_year_month", "newest_update_date"}
+
+# is_available() and stats() are cheap after the first probe and are excluded.
+
+
+def _async_callers_of_blocking_sidecar_fns():
+    """Every `local_meta.<blocking>()` call lexically inside an async def."""
+    offenders = []
+    for path in sorted(pathlib.Path("app").rglob("*.py")):
+        if path.name == "local_meta.py":
+            continue
+        tree = ast.parse(path.read_text(), filename=str(path))
+        for fn in ast.walk(tree):
+            if not isinstance(fn, ast.AsyncFunctionDef):
+                continue
+            for node in ast.walk(fn):
+                if not isinstance(node, ast.Call):
+                    continue
+                f = node.func
+                if (isinstance(f, ast.Attribute) and f.attr in BLOCKING
+                        and isinstance(f.value, ast.Name)
+                        and f.value.id == "local_meta"):
+                    offenders.append(f"{path}:{node.lineno} in {fn.name}()")
+    return offenders
+
+
+def test_no_coroutine_calls_the_sidecar_directly():
+    offenders = _async_callers_of_blocking_sidecar_fns()
+    assert not offenders, (
+        "sidecar I/O called directly from a coroutine — wrap in "
+        "asyncio.to_thread: " + ", ".join(offenders))
+
+
+def test_the_check_would_catch_a_direct_call():
+    """Guard against the guard silently matching nothing."""
+    src = (
+        "import asyncio\n"
+        "from app import local_meta\n"
+        "async def handler(ids):\n"
+        "    return local_meta.fetch_rows(ids)\n"
+    )
+    tree = ast.parse(src)
+    hits = [
+        n.lineno for fn in ast.walk(tree) if isinstance(fn, ast.AsyncFunctionDef)
+        for n in ast.walk(fn)
+        if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
+        and n.func.attr in BLOCKING
+    ]
+    assert hits == [4]
+
+
+async def test_a_threaded_query_does_not_stall_the_loop():
+    """The property itself, not just the call shape."""
+    with tempfile.TemporaryDirectory() as d:
+        path = f"{d}/t.db"
+        c = sqlite3.connect(path)
+        c.execute("CREATE TABLE papers (arxiv_id TEXT, blob TEXT)")
+        c.executemany("INSERT INTO papers VALUES (?,?)",
+                      [(f"p{i}", "x" * 200) for i in range(120_000)])
+        c.commit()
+        c.close()
+        conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True,
+                               check_same_thread=False)
+
+        def slow():
+            return conn.execute(
+                "SELECT COUNT(*) FROM papers WHERE blob LIKE '%zz%'").fetchone()
+
+        gaps = []
+        stop = asyncio.Event()
+
+        async def heartbeat():
+            last = time.perf_counter()
+            while not stop.is_set():
+                await asyncio.sleep(0.005)
+                now = time.perf_counter()
+                gaps.append(now - last)
+                last = now
+
+        hb = asyncio.create_task(heartbeat())
+        await asyncio.sleep(0.03)
+        await asyncio.to_thread(slow)
+        stop.set()
+        await hb
+        conn.close()
+
+    assert max(gaps) < 0.05, (
+        f"loop stalled {max(gaps)*1000:.0f} ms even through a thread")