on_the_way / rivinity /web /cache.py
RANITBAG's picture
3rd wave
88d1007 verified
Raw History Blame Contribute Delete
8.34 kB
"""SQLite-backed web cache with TTL, invalidation and a hard size bound.
Cache identity (the key) layers every freshness-relevant dimension:
canonical URL | crawler backend | backend version | policy version |
sanitizer version | schema version. Any change in any layer produces a new
key, so stale-format entries are never served silently.
Stored per entry: timestamp, page status, content hash, cleaned content,
metadata (JSON), HTTP status, language, final URL, title, links, redirects.
Stale contract: get() NEVER returns expired content unless the caller
explicitly passes allow_stale=True -- and even then the entry carries
stale=True and its age, so it can never be mistaken for current. The
crawler never passes allow_stale. Blocked/error pages are never stored
(H21-7: a denial must not be re-served as content on a later hit).
"""
from __future__ import annotations
import hashlib
import json
import sqlite3
import time
import urllib.parse
CACHE_SCHEMA_VERSION = 2
DEFAULT_TTL_S = 24 * 3600
DEFAULT_MAX_ENTRIES = 10_000
_COLUMNS = ["key", "url", "backend", "retrieved_at", "status", "http_status",
"final_url", "title", "text", "language", "content_hash",
"metadata", "links", "redirect_chain"]
class WebCache:
def __init__(self, path: str = ":memory:", ttl_s: float = DEFAULT_TTL_S,
max_entries: int = DEFAULT_MAX_ENTRIES):
self.ttl_s = ttl_s
self.max_entries = max_entries
self._db = sqlite3.connect(path)
self._db.execute(
"CREATE TABLE IF NOT EXISTS web_cache ("
"key TEXT PRIMARY KEY, url TEXT, backend TEXT, "
"retrieved_at REAL, status TEXT, http_status INTEGER, "
"final_url TEXT, title TEXT, text TEXT, language TEXT, "
"content_hash TEXT, metadata TEXT, links TEXT, redirect_chain TEXT)")
self._db.commit()
self._migrate()
def _migrate(self):
"""Add columns introduced after a dev DB was created (ALTER TABLE
ADD COLUMN is safe/idempotent behind a PRAGMA check)."""
existing = {r[1] for r in self._db.execute("PRAGMA table_info(web_cache)")}
for col in _COLUMNS:
if col not in existing:
self._db.execute(
f"ALTER TABLE web_cache ADD COLUMN {col} TEXT DEFAULT ''")
self._db.commit()
# ------------------------------------------------------------------ #
@staticmethod
def canonicalize(url: str) -> str:
p = urllib.parse.urlparse(url)
scheme = p.scheme.lower()
host = (p.hostname or "").lower()
port = p.port
netloc = host + (f":{port}" if port and port not in (80, 443) else "")
path = p.path or "/"
return urllib.parse.urlunparse((scheme, netloc, path, "", p.query, ""))
def key(self, url: str, backend: str, backend_version: str,
policy_version: str = "1", sanitizer_version: str = "1") -> str:
"""Cache identity: canonical URL + backend + versions."""
raw = "|".join([self.canonicalize(url), backend, backend_version,
policy_version, sanitizer_version,
f"schema{CACHE_SCHEMA_VERSION}"])
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
# ------------------------------------------------------------------ #
def get(self, url: str, backend: str, backend_version: str,
allow_stale: bool = False):
"""Return (entry_dict, fresh). Expired entries are returned ONLY with
allow_stale=True and always carry stale=True + age metadata."""
row = self._db.execute(
"SELECT url, backend, retrieved_at, status, http_status, "
"final_url, title, text, language, content_hash, metadata, "
"links, redirect_chain FROM web_cache WHERE key=?",
(self.key(url, backend, backend_version),)).fetchone()
if row is None:
return None, False
age = time.time() - row[2]
fresh = age <= self.ttl_s
if not fresh and not allow_stale:
return None, False
return {"requested_url": row[0], "backend": row[1],
"retrieved_at": row[2], "status": row[3],
"http_status": row[4], "final_url": row[5], "title": row[6],
"text": row[7], "language": row[8] or None,
"content_hash": row[9] or None,
"metadata": json.loads(row[10] or "{}"),
"links": json.loads(row[11] or "[]"),
"redirect_chain": json.loads(row[12] or "[]"),
"age_s": age, "stale": not fresh}, fresh
def put(self, page, backend_version: str, metadata: dict | None = None) -> str:
"""Store a crawled page. Returns the content hash. Blocked/error
pages are never persisted (H21-7)."""
if getattr(page, "status", "ok") != "ok":
return ""
content_hash = hashlib.sha256(
(getattr(page, "text", "") or "").encode("utf-8")).hexdigest()
md = {"backend_version": backend_version, **(metadata or {})}
self._db.execute(
"INSERT OR REPLACE INTO web_cache (key,url,backend,retrieved_at,"
"status,http_status,final_url,title,text,language,content_hash,"
"metadata,links,redirect_chain) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(self.key(page.requested_url, page.backend, backend_version),
page.requested_url, page.backend, time.time(),
getattr(page, "status", "ok"), page.http_status, page.final_url,
page.title, page.text, getattr(page, "language", None),
content_hash, json.dumps(md, ensure_ascii=False),
json.dumps(page.links, ensure_ascii=False),
json.dumps(page.redirect_chain, ensure_ascii=False)))
self._db.commit()
self._enforce_bound()
return content_hash
# ------------------------------------------------------------------ #
def invalidate(self, url: str, backend: str | None = None,
backend_version: str | None = None) -> int:
"""Explicit invalidation for one URL. backend given without a version
removes ALL versions of that backend (H21-7 fix: the old 'version 0'
key never matched stored entries, making invalidation a silent no-op)."""
if backend is not None and backend_version is not None:
cur = self._db.execute("DELETE FROM web_cache WHERE key=?",
(self.key(url, backend, backend_version),))
elif backend is not None:
cur = self._db.execute(
"DELETE FROM web_cache WHERE url=? AND backend=?",
(self.canonicalize(url), backend))
else:
cur = self._db.execute("DELETE FROM web_cache WHERE url=?",
(self.canonicalize(url),))
self._db.commit()
return cur.rowcount
def invalidate_domain(self, domain: str) -> int:
cur = self._db.execute("DELETE FROM web_cache WHERE url LIKE ?",
(f"%//{domain}/%",))
self._db.commit()
return cur.rowcount
def purge_expired(self) -> int:
cutoff = time.time() - self.ttl_s
cur = self._db.execute("DELETE FROM web_cache WHERE retrieved_at < ?",
(cutoff,))
self._db.commit()
return cur.rowcount
def _enforce_bound(self):
"""Hard bound: evict oldest-retrieved entries beyond max_entries."""
n = self._db.execute("SELECT COUNT(*) FROM web_cache").fetchone()[0]
if n > self.max_entries:
excess = n - self.max_entries
self._db.execute(
"DELETE FROM web_cache WHERE key IN (SELECT key FROM web_cache "
"ORDER BY retrieved_at ASC LIMIT ?)", (excess,))
self._db.commit()
def close(self) -> None:
"""Release the SQLite connection (required on Windows before the
underlying file can be deleted/moved)."""
self._db.close()
def stats(self) -> dict:
n = self._db.execute("SELECT COUNT(*) FROM web_cache").fetchone()[0]
return {"entries": n, "max_entries": self.max_entries,
"ttl_s": self.ttl_s, "schema_version": CACHE_SCHEMA_VERSION}