Run the blocking DuckDuckGo call in a worker thread

`DDGS.text()` is a synchronous method doing blocking HTTP I/O, and it was
being called directly from an `async def`. That froze the entire event loop
for the duration of every DuckDuckGo request: no other request progressed,
no in-flight Playwright scrape progressed, nothing else on the loop got a
turn.

DuckDuckGo is the first backend tried for every query in `/serp/search`,
which is the flagship MCP tool, and both the endpoint docstrings and
MCP_INSTRUCTIONS tell clients to batch their query variations into a single
call - so the advertised concurrency was not just absent on the primary
path, it was inverted into N sequential stalls.

Measured before the fix, with a 0.25s blocking stand-in: four gathered
queries took 1.00s (fully serialized) and the event loop got 0 turns.
After: the same four take about as long as one, and the loop keeps ticking
throughout.

tests/test_serp_ddg_concurrency.py asserts both properties directly - the
gathered queries don't serialize, and a heartbeat task keeps getting turns
while a query is in flight - so a future change that puts the blocking call
back on the loop fails the suite rather than silently costing throughput.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MSNSYnceqvdVz7Csis4K9e

serp.py

@@ -8,6 +8,7 @@ from urllib.parse import quote_plus
 import logging
 import re
 from lxml import etree
+import asyncio
 from asyncio import Semaphore
 
 # Concurrency limit for Playwright browser contexts.
@@ -277,14 +278,30 @@ async def query_bing_search(browser: Browser, q: str, n_results: int = 10):
         return await _extract_bing_results(page, n_results)
 
 
+def _ddg_text_blocking(q: str, n_results: int) -> list[dict]:
+    """The synchronous half of a DuckDuckGo query.
+
+    `DDGS.text()` performs blocking HTTP I/O. It is deliberately isolated
+    here so `query_ddg_search` can hand it to a worker thread rather than
+    calling it on the event loop.
+    """
+    return list(DDGS().text(q, max_results=n_results))
+
+
 async def query_ddg_search(q: str, n_results: int = 10):
-    """Queries duckduckgo search for the specified query"""
-    ddgs = DDGS()
-    results = []
+    """Queries duckduckgo search for the specified query.
+
+    The underlying library call is synchronous, so it runs in a worker
+    thread: calling it directly would freeze the whole event loop for the
+    duration of the request, and DuckDuckGo is the first backend tried for
+    every query in `/serp/search`.
+    """
+    raw_results = await asyncio.to_thread(_ddg_text_blocking, q, n_results)
 
-    for result in ddgs.text(q, max_results=n_results):
-        results.append(
-            {"title": result["title"], "body": result["body"], "href": result["href"]})
+    results = [
+        {"title": r["title"], "body": r["body"], "href": r["href"]}
+        for r in raw_results
+    ]
 
     if not results:
         raise DuckDuckGoBlockedException()

tests/test_serp_ddg_concurrency.py

@@ -0,0 +1,72 @@
+"""The DuckDuckGo backend must not block the event loop.
+
+`DDGS.text()` is a synchronous, blocking method. Calling it directly from
+an `async def` freezes the whole process for the duration of the HTTP
+request: no other request progresses, no in-flight Playwright scrape
+progresses, and nothing else on the loop gets a turn.
+
+That matters more here than it looks. DuckDuckGo is the *first* backend
+tried for every query in `/serp/search`, which is the flagship MCP tool,
+and both the endpoint docstrings and MCP_INSTRUCTIONS actively encourage
+clients to batch many queries into one call - which under a blocking
+implementation serializes into N sequential stalls instead of N concurrent
+requests.
+
+These tests use a blocking stand-in with a known duration rather than the
+real library, so they assert the concurrency property itself and stay fast
+and offline.
+"""
+
+import asyncio
+import time
+
+import serp
+from serp import query_ddg_search
+
+BLOCK_SECONDS = 0.25
+
+
+class _BlockingDDGS:
+    """Stands in for the real DDGS: a synchronous call that takes real time."""
+
+    def text(self, q, max_results=10):
+        time.sleep(BLOCK_SECONDS)
+        return [{"title": q, "body": "b", "href": "h"}]
+
+
+async def test_concurrent_queries_do_not_serialize(monkeypatch):
+    """Four gathered queries should take about as long as one, not four."""
+    monkeypatch.setattr(serp, "DDGS", lambda: _BlockingDDGS())
+
+    started = time.perf_counter()
+    await asyncio.gather(*[query_ddg_search(f"q{i}", 5) for i in range(4)])
+    elapsed = time.perf_counter() - started
+
+    assert elapsed < BLOCK_SECONDS * 2, (
+        f"4 concurrent queries took {elapsed:.2f}s; "
+        f"serialized would be ~{BLOCK_SECONDS * 4:.2f}s, concurrent ~{BLOCK_SECONDS:.2f}s")
+
+
+async def test_the_event_loop_stays_responsive_during_a_query(monkeypatch):
+    """Anything else scheduled on the loop must keep getting turns while a
+    DuckDuckGo query is in flight - this is what a blocked loop costs every
+    other request being served at the same time.
+    """
+    monkeypatch.setattr(serp, "DDGS", lambda: _BlockingDDGS())
+
+    ticks = 0
+    stop = asyncio.Event()
+
+    async def heartbeat():
+        nonlocal ticks
+        while not stop.is_set():
+            await asyncio.sleep(0.01)
+            ticks += 1
+
+    beat = asyncio.create_task(heartbeat())
+    await query_ddg_search("widget", 5)
+    stop.set()
+    await beat
+
+    # A responsive loop ticks ~25 times in 0.25s; a blocked one manages ~1.
+    assert ticks > 10, f"event loop only got {ticks} turns during a {BLOCK_SECONDS}s query"