Stop reporting upstream failures as "patent not found"

`scrap_patent` caught HTTPStatusError and bare Exception identically and
fell through to a single 404, so every failure mode collapsed into "not
found": a 503 from Google Patents, a connect timeout, and an unparseable
interstitial all told the caller the patent does not exist.

That is worse than an inaccurate status code here, because the MCP
instructions this server ships tell the model in as many words: "A patent
that returns 'not found' is genuinely absent from every backend; move on
to the next id rather than retrying." Correct advice given a truthful 404,
actively harmful given this one - a transient blip taught an agent, with
the server's explicit encouragement, that a real patent does not exist.

Now 404 is reserved for a backend positively saying the document is
absent, and is only returned when *both* backends said so (or when the
only backend asked did). Upstream failures map to 502, timeouts to 504.
The same mapping is applied to the direct /ops endpoint.

tests/test_patent_fallback.py covers the whole matrix, which previously
had no tests at all: the happy path skipping OPS entirely, a real 404,
four upstream error statuses, a timeout, an unparseable page, the fallback
succeeding after Google Patents fails, both backends missing, and OPS
failing to answer after a Google Patents 404 - unknown, not absent.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MSNSYnceqvdVz7Csis4K9e

app.py

@@ -275,6 +275,26 @@ async def search(params: SerpQuery):
 # =========================== Scrapping endpoints ===========================
 
 
+def _is_not_found(exc: Exception) -> bool:
+    """True only when a backend positively said the document is absent.
+
+    Anything else - a 5xx, a rate-limit, a timeout, an unparseable page -
+    is a failure to answer, not an answer.
+    """
+    return isinstance(exc, HTTPStatusError) and exc.response.status_code == 404
+
+
+def _upstream_status(exc: Exception) -> int:
+    """The status to report for a failed upstream call. Never 404.
+
+    404 is a claim that the patent does not exist, and MCP_INSTRUCTIONS
+    tells agents to believe it and move on without retrying - so it has to
+    be reserved for a backend actually saying so. A transient outage
+    reported as 404 teaches an agent that a real patent is absent.
+    """
+    return 504 if isinstance(exc, httpx.TimeoutException) else 502
+
+
 @scrap_router.get("/scrap_patent/{patent_id}")
 async def scrap_patent(patent_id: PatentId) -> PatentScrapResult:
     """Scraps the specified patent from Google Patents.
@@ -282,34 +302,42 @@ async def scrap_patent(patent_id: PatentId) -> PatentScrapResult:
     Falls back to the EPO OPS API (which covers patents missing from Google
     Patents) when the scrape fails and OPS credentials are configured.
     """
+    google_error: Exception
     try:
         return await scrap_patent_async(httpx_client, f"https://patents.google.com/patent/{patent_id}/en")
     except HTTPStatusError as e:
+        google_error = e
         logging.warning(
             f"Google Patents returned {e.response.status_code} for {patent_id}.")
     except Exception as e:
+        google_error = e
         logging.warning(f"Failed to scrap patent {patent_id}: {e}")
 
     if not ops_token_manager.configured:
+        if _is_not_found(google_error):
+            raise HTTPException(
+                status_code=404,
+                detail=f"Patent '{patent_id}' not found on Google Patents (EPO OPS fallback not configured).")
         raise HTTPException(
-            status_code=404,
-            detail=f"Patent '{patent_id}' not found on Google Patents (EPO OPS fallback not configured).")
+            status_code=_upstream_status(google_error),
+            detail=f"Google Patents request failed for '{patent_id}' and the EPO OPS fallback is not configured: {google_error}")
 
     try:
         logging.info(f"Trying OPS for patent {patent_id}.")
         return await ops_scrap_patent(httpx_client, patent_id)
-    except HTTPStatusError as e:
-        if e.response.status_code == 404:
+    except Exception as e:
+        logging.warning(f"OPS fallback failed for {patent_id}: {e}")
+        # Only claim the patent doesn't exist when *both* backends said so.
+        if _is_not_found(google_error) and _is_not_found(e):
             raise HTTPException(
                 status_code=404,
                 detail=f"Patent '{patent_id}' not found on Google Patents or EPO OPS.")
+        if isinstance(e, HTTPStatusError):
+            raise HTTPException(
+                status_code=_upstream_status(e),
+                detail=f"EPO OPS returned {e.response.status_code} for '{patent_id}'.")
         raise HTTPException(
-            status_code=502,
-            detail=f"EPO OPS returned {e.response.status_code} for '{patent_id}'.")
-    except Exception as e:
-        logging.warning(f"OPS fallback failed for {patent_id}: {e}")
-        raise HTTPException(
-            status_code=502,
+            status_code=_upstream_status(e),
             detail=f"EPO OPS request failed for '{patent_id}': {e}")
 
 
@@ -360,12 +388,13 @@ async def ops_get_patent(patent_id: PatentId) -> PatentScrapResult:
             raise HTTPException(
                 status_code=404, detail=f"Patent '{patent_id}' not found in EPO OPS.")
         raise HTTPException(
-            status_code=502,
+            status_code=_upstream_status(e),
             detail=f"EPO OPS returned {e.response.status_code} for '{patent_id}'.")
     except Exception as e:
         logging.warning(f"Failed to retrieve patent {patent_id} from OPS: {e}")
         raise HTTPException(
-            status_code=502, detail=f"EPO OPS request failed for '{patent_id}': {e}")
+            status_code=_upstream_status(e),
+            detail=f"EPO OPS request failed for '{patent_id}': {e}")
 
 
 @ops_router.post("/scrap_patents_bulk", response_model=OPSBulkResponse)

tests/test_patent_fallback.py

@@ -0,0 +1,195 @@
+"""The Google Patents -> EPO OPS fallback chain in app.py.
+
+This is the most heavily-branched code in the module - scrape, catch,
+check whether OPS is configured, fall back, then map OPS's own failures
+onto status codes - and it had no test coverage at all.
+
+The distinction these tests exist to protect: a 404 means "this patent
+does not exist", and an agent is explicitly told by MCP_INSTRUCTIONS to
+believe it and move on without retrying. Anything else - an upstream
+outage, a timeout, a blocked scrape - must not be reported that way.
+"""
+
+import httpx
+import pytest
+from httpx import ASGITransport, HTTPStatusError, Request, Response
+
+import app as app_module
+from scrap import PatentScrapResult
+
+PATENT_ID = "US11930446B2"
+
+
+@pytest.fixture
+async def client():
+    transport = ASGITransport(app=app_module.app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
+        yield c
+
+
+@pytest.fixture
+def ops_configured(monkeypatch):
+    monkeypatch.setattr(app_module.ops_token_manager, "_key", "k")
+    monkeypatch.setattr(app_module.ops_token_manager, "_secret", "s")
+
+
+@pytest.fixture
+def ops_unconfigured(monkeypatch):
+    monkeypatch.setattr(app_module.ops_token_manager, "_key", None)
+    monkeypatch.setattr(app_module.ops_token_manager, "_secret", None)
+
+
+def _http_error(status: int) -> HTTPStatusError:
+    request = Request("GET", "https://patents.google.com/")
+    return HTTPStatusError(
+        str(status), request=request, response=Response(status, request=request))
+
+
+def _raises(exc):
+    async def _fn(*args, **kwargs):
+        raise exc
+    return _fn
+
+
+# ------------------------------- the happy path -------------------------------
+
+
+async def test_successful_scrape_never_touches_ops(client, ops_configured, monkeypatch):
+    ops_called = False
+
+    async def fake_ops(*args, **kwargs):
+        nonlocal ops_called
+        ops_called = True
+        return PatentScrapResult(title="from OPS")
+
+    monkeypatch.setattr(
+        app_module, "scrap_patent_async",
+        lambda c, url: _ok(PatentScrapResult(title="from Google Patents")))
+    monkeypatch.setattr(app_module, "ops_scrap_patent", fake_ops)
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 200
+    assert resp.json()["title"] == "from Google Patents"
+    assert ops_called is False
+
+
+async def _ok(value):
+    return value
+
+
+# ------------------------- genuine miss vs upstream failure -------------------------
+
+
+async def test_a_real_404_from_google_patents_is_a_404(client, ops_unconfigured, monkeypatch):
+    """The one case where "not found" is the truth."""
+    monkeypatch.setattr(app_module, "scrap_patent_async", _raises(_http_error(404)))
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 404
+
+
+@pytest.mark.parametrize("status", [429, 500, 502, 503])
+async def test_an_upstream_failure_is_not_reported_as_not_found(
+        client, ops_unconfigured, monkeypatch, status):
+    """MCP_INSTRUCTIONS tells the model a "not found" patent is genuinely
+    absent everywhere and to move on rather than retrying. Reporting a
+    transient upstream failure that way teaches an agent - with the
+    server's explicit encouragement - that a real patent does not exist.
+    """
+    monkeypatch.setattr(app_module, "scrap_patent_async", _raises(_http_error(status)))
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 502, f"upstream {status} was reported as {resp.status_code}"
+
+
+async def test_a_timeout_is_reported_as_a_gateway_timeout(client, ops_unconfigured, monkeypatch):
+    monkeypatch.setattr(
+        app_module, "scrap_patent_async", _raises(httpx.ConnectTimeout("timed out")))
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 504
+
+
+async def test_an_unparseable_page_is_reported_as_a_bad_gateway(
+        client, ops_unconfigured, monkeypatch):
+    """parse_patent_html raises ValueError when the page isn't a patent page
+    (an interstitial, or a markup change). That is our problem or theirs,
+    but it is not evidence the patent doesn't exist.
+    """
+    monkeypatch.setattr(
+        app_module, "scrap_patent_async", _raises(ValueError("no DC.title")))
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 502
+
+
+# --------------------------------- the OPS fallback ---------------------------------
+
+
+async def test_falls_back_to_ops_when_google_patents_fails(client, ops_configured, monkeypatch):
+    monkeypatch.setattr(app_module, "scrap_patent_async", _raises(_http_error(503)))
+    monkeypatch.setattr(
+        app_module, "ops_scrap_patent",
+        lambda c, pid: _ok(PatentScrapResult(title="from OPS")))
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 200
+    assert resp.json()["title"] == "from OPS"
+
+
+async def test_missing_from_both_backends_is_a_404(client, ops_configured, monkeypatch):
+    monkeypatch.setattr(app_module, "scrap_patent_async", _raises(_http_error(404)))
+    monkeypatch.setattr(app_module, "ops_scrap_patent", _raises(_http_error(404)))
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 404
+    assert "not found" in resp.json()["detail"].lower()
+
+
+async def test_ops_failing_after_a_google_patents_404_is_not_a_404(
+        client, ops_configured, monkeypatch):
+    """Google Patents says the patent is missing, but OPS - the backend that
+    covers what Google Patents doesn't - never answered. That is unknown,
+    not absent.
+    """
+    monkeypatch.setattr(app_module, "scrap_patent_async", _raises(_http_error(404)))
+    monkeypatch.setattr(app_module, "ops_scrap_patent", _raises(_http_error(500)))
+
+    resp = await client.get(f"/scrap/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 502
+
+
+# ------------------------------ the direct /ops endpoint ------------------------------
+
+
+async def test_ops_endpoint_reports_a_timeout_as_a_gateway_timeout(
+        client, ops_configured, monkeypatch):
+    monkeypatch.setattr(
+        app_module, "ops_scrap_patent", _raises(httpx.ConnectTimeout("timed out")))
+
+    resp = await client.get(f"/ops/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 504
+
+
+async def test_ops_endpoint_reports_a_real_miss_as_404(client, ops_configured, monkeypatch):
+    """Here OPS is the only backend asked, so its 404 is the whole answer."""
+    monkeypatch.setattr(app_module, "ops_scrap_patent", _raises(_http_error(404)))
+
+    resp = await client.get(f"/ops/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 404
+
+
+async def test_ops_endpoint_is_503_when_not_configured(client, ops_unconfigured):
+    resp = await client.get(f"/ops/scrap_patent/{PATENT_ID}")
+
+    assert resp.status_code == 503