fix(link-rot): validate PDF candidates + untrack llm_health.json

THE INCIDENT
Overnight 2026-05-27/28 the nightly link-rot LaunchAgent
(com.rohit.insurancebot.linkrot → tools/check_link_rot.py) applied
seven bad auto-patches to 40-data/corpus_urls.md and one to
tools/.pdf_etag_state.json:

- HDFC ERGO ×3 + Bajaj Allianz ×2 + New India ×1: rewrote https→http
via try_canonicalise — both work but the downgrade is gratuitous.
- Niva Bupa Health Companion (a core indemnity product): rewrote the
live URL to a 2-year-old Wayback wrapper that serves HTML, not PDF.
Would have silently broken corpus rebuild for that policy.

ROOT CAUSE
head_check() did HEAD only; the affected origins (HDFC ERGO, Bajaj,
New India, Niva Bupa) return text/html interstitials or transient
errors for HEAD while serving the real PDF on GET. When HEAD failed
transiently, the tool fell into auto_fix and both strategies (try_
canonicalise + try_wayback) accepted any candidate whose HEAD returned
status 2xx — without verifying it ACTUALLY served a PDF. The Wayback
'available' API returns the HTML-wrapped /web/<ts>/ URL by default,
which is 200 OK but text/html.

THE FIX
1. New _validate_pdf_candidate(url): GET Range bytes=0-31, accept only
if status ∈ {200,206} AND (body starts with %PDF OR content-type
contains 'pdf'). Some CDNs send octet-stream for PDFs, so the magic
check is the strict gate; content-type is the loose one.
2. New _looks_like_pdf_url(url): detects .pdf paths and Wayback-wrapped
.pdf originals — only those URLs require PDF validation (HTML
review URLs etc. still flow through unchanged).
3. New _wayback_raw_variant(url): rewrites
https://web.archive.org/web/<ts>/<orig> →
https://web.archive.org/web/<ts>id_/<orig> so the archive serves
the original PDF bytes (no HTML wrapper).
4. try_canonicalise now requires expected-PDF candidates to pass
_validate_pdf_candidate before being returned as a replacement.
5. try_wayback now rewrites to id_ form AND validates before returning.

VERIFIED on the exact problem URLs (live probes):
Niva Bupa LIVE True ← no fallback needed
Niva Bupa Wayback wrap False ← the bad fallback the old code took
Niva Bupa Wayback id_ True ← the new fallback works
HDFC ERGO https True
HDFC ERGO http True (both work; canonicalise no longer
chosen unless https first fails)
NewIndia https True
NewIndia http False ← would be rejected as bad fallback

ALSO: untrack 40-data/llm_health.json
The file is process-local probe state, written on every probe tick.
Local-dev probebacks (e.g. SSL CERTIFICATE_VERIFY_FAILED rows from
running outside the HF Space) would otherwise stage 100s of stale
'down' lines that overwrite the live snapshot on next push. Backend
tolerates missing (llm_health.py:389) and rebuilds chains/candidates
on first probe after startup.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.gitignore

@@ -91,6 +91,12 @@ README.pdf
 # Runtime + heavy binary: never commit
 40-data/policies/
 40-data/sessions/
+# Process-local LLM probe snapshot — written by backend/llm_health.py on
+# every probe tick. Local-dev probebacks would otherwise stage hundreds
+# of stale "down" rows that overwrite the HF Space's live snapshot. The
+# backend tolerates missing (llm_health.py:389) and rebuilds
+# chains/candidates/recent_turns on first probe after startup.
+40-data/llm_health.json
 80-audit/full_*/
 80-audit/postfix_*/
 MUST_FIX.md

40-data/llm_health.json

@@ -1,381 +0,0 @@
-{
-  "updated_at": "2026-05-16T20:23:07Z",
-  "models": {
-    "qwen/qwen3-next-80b-a3b-instruct": {
-      "model": "qwen/qwen3-next-80b-a3b-instruct",
-      "status": "healthy",
-      "last_success_at": "2026-05-16T20:23:07Z",
-      "last_failure_at": "2026-05-16T17:55:07Z",
-      "last_error": null,
-      "last_status_code": null,
-      "latency_ms": 907,
-      "consecutive_failures": 0,
-      "tested_at": "2026-05-16T20:23:07Z",
-      "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 1145,
-          "ts": "2026-05-16T20:00:14Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 658,
-          "ts": "2026-05-16T20:06:08Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 1416,
-          "ts": "2026-05-16T20:11:26Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 499,
-          "ts": "2026-05-16T20:17:37Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 907,
-          "ts": "2026-05-16T20:23:07Z",
-          "src": "probe"
-        }
-      ],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "qwen/qwen3.5-122b-a10b": {
-      "model": "qwen/qwen3.5-122b-a10b",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "timeout",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "openai/gpt-oss-120b": {
-      "model": "openai/gpt-oss-120b",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "empty_content",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "mistralai/mistral-large-3-675b-instruct-2512": {
-      "model": "mistralai/mistral-large-3-675b-instruct-2512",
-      "status": "down",
-      "last_success_at": "2026-05-16T19:28:36Z",
-      "last_failure_at": "2026-05-16T20:23:07Z",
-      "last_error": "timeout",
-      "last_status_code": null,
-      "latency_ms": 5119,
-      "consecutive_failures": 9,
-      "tested_at": "2026-05-16T20:23:07Z",
-      "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 8004,
-          "ts": "2026-05-16T20:00:14Z",
-          "src": "probe"
-        },
-        {
-          "ok": false,
-          "latency_ms": 42670,
-          "ts": "2026-05-16T20:06:08Z",
-          "src": "probe"
-        },
-        {
-          "ok": false,
-          "latency_ms": 8009,
-          "ts": "2026-05-16T20:11:26Z",
-          "src": "probe"
-        },
-        {
-          "ok": false,
-          "latency_ms": 8003,
-          "ts": "2026-05-16T20:17:37Z",
-          "src": "probe"
-        },
-        {
-          "ok": false,
-          "latency_ms": 7997,
-          "ts": "2026-05-16T20:23:07Z",
-          "src": "probe"
-        }
-      ],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "nvidia/llama-3.3-nemotron-super-49b-v1.5": {
-      "model": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
-      "status": "healthy",
-      "last_success_at": "2026-05-16T20:23:07Z",
-      "last_failure_at": "2026-05-16T20:17:37Z",
-      "last_error": null,
-      "last_status_code": null,
-      "latency_ms": 296,
-      "consecutive_failures": 0,
-      "tested_at": "2026-05-16T20:23:07Z",
-      "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 280,
-          "ts": "2026-05-16T20:00:14Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 388,
-          "ts": "2026-05-16T20:06:08Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 306,
-          "ts": "2026-05-16T20:11:26Z",
-          "src": "probe"
-        },
-        {
-          "ok": false,
-          "latency_ms": 62582,
-          "ts": "2026-05-16T20:17:37Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 296,
-          "ts": "2026-05-16T20:23:07Z",
-          "src": "probe"
-        }
-      ],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "meta/llama-3.3-70b-instruct": {
-      "model": "meta/llama-3.3-70b-instruct",
-      "status": "healthy",
-      "last_success_at": "2026-05-14T16:35:39Z",
-      "last_failure_at": "2026-05-14T15:58:40Z",
-      "last_error": null,
-      "last_status_code": null,
-      "latency_ms": 479,
-      "consecutive_failures": 0,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "deepseek-ai/deepseek-v4-pro": {
-      "model": "deepseek-ai/deepseek-v4-pro",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "timeout",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "nvidia/nemotron-3-nano-30b-a3b": {
-      "model": "nvidia/nemotron-3-nano-30b-a3b",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "empty_content",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "deepseek-ai/deepseek-v4-flash": {
-      "model": "deepseek-ai/deepseek-v4-flash",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "timeout",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "moonshotai/kimi-k2-instruct-0905": {
-      "model": "moonshotai/kimi-k2-instruct-0905",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "http_404",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "minimaxai/minimax-m2.5": {
-      "model": "minimaxai/minimax-m2.5",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "http_410",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "meta/llama-4-maverick-17b-128e-instruct": {
-      "model": "meta/llama-4-maverick-17b-128e-instruct",
-      "status": "healthy",
-      "last_success_at": "2026-05-16T20:23:07Z",
-      "last_failure_at": "2026-05-16T14:04:03Z",
-      "last_error": null,
-      "last_status_code": null,
-      "latency_ms": 1302,
-      "consecutive_failures": 0,
-      "tested_at": "2026-05-16T20:23:07Z",
-      "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 1555,
-          "ts": "2026-05-16T20:00:14Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 969,
-          "ts": "2026-05-16T20:06:08Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 900,
-          "ts": "2026-05-16T20:11:26Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 564,
-          "ts": "2026-05-16T20:17:37Z",
-          "src": "probe"
-        },
-        {
-          "ok": true,
-          "latency_ms": 1302,
-          "ts": "2026-05-16T20:23:07Z",
-          "src": "probe"
-        }
-      ],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    },
-    "openrouter:openai/gpt-oss-120b": {
-      "model": "openrouter:openai/gpt-oss-120b",
-      "status": "down",
-      "last_success_at": null,
-      "last_failure_at": "2026-05-14T16:35:39Z",
-      "last_error": "empty_content",
-      "last_status_code": null,
-      "latency_ms": null,
-      "consecutive_failures": 4,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": 32.802310819,
-      "credits_unit": "usd_balance",
-      "credits_reset_at": null,
-      "credits_observed_at": 97811.656528458,
-      "credits_low_water": 0.05
-    },
-    "groq:llama-3.3-70b-versatile": {
-      "model": "groq:llama-3.3-70b-versatile",
-      "status": "healthy",
-      "last_success_at": "2026-05-14T16:35:39Z",
-      "last_failure_at": null,
-      "last_error": null,
-      "last_status_code": null,
-      "latency_ms": 200,
-      "consecutive_failures": 0,
-      "tested_at": "2026-05-14T16:35:39Z",
-      "probe_history": [],
-      "degraded_until_monotonic": 0.0,
-      "credits_remaining": null,
-      "credits_unit": null,
-      "credits_reset_at": null,
-      "credits_observed_at": null,
-      "credits_low_water": 0.0
-    }
-  }
-}

tools/check_link_rot.py

@@ -163,8 +163,63 @@ def head_check(url: str, client: httpx.Client) -> tuple[int, str]:
 # ---------- auto-fix strategies ----------
 
 
+def _looks_like_pdf_url(url: str) -> bool:
+    """Heuristic: corpus expects a PDF at this URL."""
+    # Direct .pdf paths
+    if ".pdf" in url.lower().split("?", 1)[0]:
+        return True
+    # Wayback-wrapped originals: https://web.archive.org/web/<ts>/<original>
+    if "web.archive.org/web/" in url and ".pdf" in url.lower():
+        return True
+    return False
+
+
+def _validate_pdf_candidate(url: str, client: httpx.Client) -> bool:
+    """Confirm a candidate URL ACTUALLY serves a PDF (Bug #X 2026-05-28).
+
+    Background: HEAD on the same path can return 200 with text/html (server
+    interstitial) while GET returns 200 with %PDF — and vice versa on
+    transient failures. The previous auto-fix accepted any candidate that
+    returned 200 to a HEAD, which silently downgraded https→http and
+    swapped live URLs for HTML-only Wayback wrappers. Validate by
+    GET-Range and check the file magic.
+    """
+    try:
+        r = client.get(
+            url,
+            headers={**HEADERS, "Range": "bytes=0-31"},
+            follow_redirects=True,
+            timeout=15,
+        )
+        if r.status_code not in (200, 206):
+            return False
+        ct = (r.headers.get("Content-Type") or "").lower()
+        body = r.content[:4]
+        # Accept either explicit PDF content-type OR %PDF magic — some
+        # CDNs send octet-stream / generic types for PDF assets.
+        return body == b"%PDF" or "pdf" in ct
+    except (httpx.HTTPError, ValueError):
+        return False
+
+
+def _wayback_raw_variant(url: str) -> str:
+    """Rewrite a Wayback snapshot URL to its `id_` raw variant so the
+    archive serves the original PDF bytes instead of the HTML wrapper.
+    https://web.archive.org/web/<ts>/<orig>  -->
+    https://web.archive.org/web/<ts>id_/<orig>"""
+    m = re.match(r"^(https?://web\.archive\.org/web/\d+)(/.+)$", url)
+    if not m:
+        return url
+    return f"{m.group(1)}id_{m.group(2)}"
+
+
 def try_wayback(url: str, client: httpx.Client) -> str | None:
-    """Return a working Wayback Machine snapshot URL, or None."""
+    """Return a working Wayback Machine snapshot URL, or None.
+
+    For PDF URLs, rewrite to the `id_` raw variant and verify the
+    snapshot actually serves the PDF bytes (some snapshots are HTML
+    error pages with status 200).
+    """
     try:
         r = client.get(
             "https://archive.org/wayback/available",
@@ -175,15 +230,27 @@ def try_wayback(url: str, client: httpx.Client) -> str | None:
         if r.status_code != 200:
             return None
         snap = r.json().get("archived_snapshots", {}).get("closest", {})
-        if snap.get("available") and snap.get("status", "").startswith("2"):
-            return snap.get("url")
+        if not (snap.get("available") and snap.get("status", "").startswith("2")):
+            return None
+        candidate = snap.get("url")
+        if not candidate:
+            return None
+        if _looks_like_pdf_url(url):
+            candidate = _wayback_raw_variant(candidate)
+            if not _validate_pdf_candidate(candidate, client):
+                return None
+        return candidate
     except (httpx.HTTPError, ValueError):
         return None
-    return None
 
 
 def try_canonicalise(url: str, client: httpx.Client) -> str | None:
-    """Strip query strings, flip http<->https."""
+    """Strip query strings, flip http<->https.
+
+    For PDF URLs, the candidate MUST serve actual PDF bytes (not just
+    return HTTP 200) — HEAD alone is too credulous on servers that
+    answer interstitial HTML for HEAD but real PDF for GET.
+    """
     candidates = []
     if "?" in url:
         candidates.append(url.split("?", 1)[0])
@@ -191,10 +258,14 @@ def try_canonicalise(url: str, client: httpx.Client) -> str | None:
         candidates.append("https://" + url[len("http://") :])
     elif url.startswith("https://"):
         candidates.append("http://" + url[len("https://") :])
+    expect_pdf = _looks_like_pdf_url(url)
     for c in candidates:
         s, _ = head_check(c, client)
-        if 200 <= s < 400:
-            return c
+        if not (200 <= s < 400):
+            continue
+        if expect_pdf and not _validate_pdf_candidate(c, client):
+            continue
+        return c
     return None