"""SQLite-backed web cache with TTL, invalidation and a hard size bound. Cache identity (the key) layers every freshness-relevant dimension: canonical URL | crawler backend | backend version | policy version | sanitizer version | schema version. Any change in any layer produces a new key, so stale-format entries are never served silently. Stored per entry: timestamp, page status, content hash, cleaned content, metadata (JSON), HTTP status, language, final URL, title, links, redirects. Stale contract: get() NEVER returns expired content unless the caller explicitly passes allow_stale=True -- and even then the entry carries stale=True and its age, so it can never be mistaken for current. The crawler never passes allow_stale. Blocked/error pages are never stored (H21-7: a denial must not be re-served as content on a later hit). """ from __future__ import annotations import hashlib import json import sqlite3 import time import urllib.parse CACHE_SCHEMA_VERSION = 2 DEFAULT_TTL_S = 24 * 3600 DEFAULT_MAX_ENTRIES = 10_000 _COLUMNS = ["key", "url", "backend", "retrieved_at", "status", "http_status", "final_url", "title", "text", "language", "content_hash", "metadata", "links", "redirect_chain"] class WebCache: def __init__(self, path: str = ":memory:", ttl_s: float = DEFAULT_TTL_S, max_entries: int = DEFAULT_MAX_ENTRIES): self.ttl_s = ttl_s self.max_entries = max_entries self._db = sqlite3.connect(path) self._db.execute( "CREATE TABLE IF NOT EXISTS web_cache (" "key TEXT PRIMARY KEY, url TEXT, backend TEXT, " "retrieved_at REAL, status TEXT, http_status INTEGER, " "final_url TEXT, title TEXT, text TEXT, language TEXT, " "content_hash TEXT, metadata TEXT, links TEXT, redirect_chain TEXT)") self._db.commit() self._migrate() def _migrate(self): """Add columns introduced after a dev DB was created (ALTER TABLE ADD COLUMN is safe/idempotent behind a PRAGMA check).""" existing = {r[1] for r in self._db.execute("PRAGMA table_info(web_cache)")} for col in _COLUMNS: if col not in existing: self._db.execute( f"ALTER TABLE web_cache ADD COLUMN {col} TEXT DEFAULT ''") self._db.commit() # ------------------------------------------------------------------ # @staticmethod def canonicalize(url: str) -> str: p = urllib.parse.urlparse(url) scheme = p.scheme.lower() host = (p.hostname or "").lower() port = p.port netloc = host + (f":{port}" if port and port not in (80, 443) else "") path = p.path or "/" return urllib.parse.urlunparse((scheme, netloc, path, "", p.query, "")) def key(self, url: str, backend: str, backend_version: str, policy_version: str = "1", sanitizer_version: str = "1") -> str: """Cache identity: canonical URL + backend + versions.""" raw = "|".join([self.canonicalize(url), backend, backend_version, policy_version, sanitizer_version, f"schema{CACHE_SCHEMA_VERSION}"]) return hashlib.sha256(raw.encode("utf-8")).hexdigest() # ------------------------------------------------------------------ # def get(self, url: str, backend: str, backend_version: str, allow_stale: bool = False): """Return (entry_dict, fresh). Expired entries are returned ONLY with allow_stale=True and always carry stale=True + age metadata.""" row = self._db.execute( "SELECT url, backend, retrieved_at, status, http_status, " "final_url, title, text, language, content_hash, metadata, " "links, redirect_chain FROM web_cache WHERE key=?", (self.key(url, backend, backend_version),)).fetchone() if row is None: return None, False age = time.time() - row[2] fresh = age <= self.ttl_s if not fresh and not allow_stale: return None, False return {"requested_url": row[0], "backend": row[1], "retrieved_at": row[2], "status": row[3], "http_status": row[4], "final_url": row[5], "title": row[6], "text": row[7], "language": row[8] or None, "content_hash": row[9] or None, "metadata": json.loads(row[10] or "{}"), "links": json.loads(row[11] or "[]"), "redirect_chain": json.loads(row[12] or "[]"), "age_s": age, "stale": not fresh}, fresh def put(self, page, backend_version: str, metadata: dict | None = None) -> str: """Store a crawled page. Returns the content hash. Blocked/error pages are never persisted (H21-7).""" if getattr(page, "status", "ok") != "ok": return "" content_hash = hashlib.sha256( (getattr(page, "text", "") or "").encode("utf-8")).hexdigest() md = {"backend_version": backend_version, **(metadata or {})} self._db.execute( "INSERT OR REPLACE INTO web_cache (key,url,backend,retrieved_at," "status,http_status,final_url,title,text,language,content_hash," "metadata,links,redirect_chain) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (self.key(page.requested_url, page.backend, backend_version), page.requested_url, page.backend, time.time(), getattr(page, "status", "ok"), page.http_status, page.final_url, page.title, page.text, getattr(page, "language", None), content_hash, json.dumps(md, ensure_ascii=False), json.dumps(page.links, ensure_ascii=False), json.dumps(page.redirect_chain, ensure_ascii=False))) self._db.commit() self._enforce_bound() return content_hash # ------------------------------------------------------------------ # def invalidate(self, url: str, backend: str | None = None, backend_version: str | None = None) -> int: """Explicit invalidation for one URL. backend given without a version removes ALL versions of that backend (H21-7 fix: the old 'version 0' key never matched stored entries, making invalidation a silent no-op).""" if backend is not None and backend_version is not None: cur = self._db.execute("DELETE FROM web_cache WHERE key=?", (self.key(url, backend, backend_version),)) elif backend is not None: cur = self._db.execute( "DELETE FROM web_cache WHERE url=? AND backend=?", (self.canonicalize(url), backend)) else: cur = self._db.execute("DELETE FROM web_cache WHERE url=?", (self.canonicalize(url),)) self._db.commit() return cur.rowcount def invalidate_domain(self, domain: str) -> int: cur = self._db.execute("DELETE FROM web_cache WHERE url LIKE ?", (f"%//{domain}/%",)) self._db.commit() return cur.rowcount def purge_expired(self) -> int: cutoff = time.time() - self.ttl_s cur = self._db.execute("DELETE FROM web_cache WHERE retrieved_at < ?", (cutoff,)) self._db.commit() return cur.rowcount def _enforce_bound(self): """Hard bound: evict oldest-retrieved entries beyond max_entries.""" n = self._db.execute("SELECT COUNT(*) FROM web_cache").fetchone()[0] if n > self.max_entries: excess = n - self.max_entries self._db.execute( "DELETE FROM web_cache WHERE key IN (SELECT key FROM web_cache " "ORDER BY retrieved_at ASC LIMIT ?)", (excess,)) self._db.commit() def close(self) -> None: """Release the SQLite connection (required on Windows before the underlying file can be deleted/moved).""" self._db.close() def stats(self) -> dict: n = self._db.execute("SELECT COUNT(*) FROM web_cache").fetchone()[0] return {"entries": n, "max_entries": self.max_entries, "ttl_s": self.ttl_s, "schema_version": CACHE_SCHEMA_VERSION}