Download rivinity/web/cache.py from RANITBAG/on_the_way: direct link, hf CLI and curl.
- Browser
- Download file 8.34 kB
-
https://huggingface.co/RANITBAG/on_the_way/resolve/main/rivinity/web/cache.py
- Command line
-
hf download hf://RANITBAG/on_the_way/rivinity/web/cache.py
-
curl -L -o cache.py https://huggingface.co/RANITBAG/on_the_way/resolve/main/rivinity/web/cache.py
8.34 kB
| """SQLite-backed web cache with TTL, invalidation and a hard size bound. | |
| Cache identity (the key) layers every freshness-relevant dimension: | |
| canonical URL | crawler backend | backend version | policy version | | |
| sanitizer version | schema version. Any change in any layer produces a new | |
| key, so stale-format entries are never served silently. | |
| Stored per entry: timestamp, page status, content hash, cleaned content, | |
| metadata (JSON), HTTP status, language, final URL, title, links, redirects. | |
| Stale contract: get() NEVER returns expired content unless the caller | |
| explicitly passes allow_stale=True -- and even then the entry carries | |
| stale=True and its age, so it can never be mistaken for current. The | |
| crawler never passes allow_stale. Blocked/error pages are never stored | |
| (H21-7: a denial must not be re-served as content on a later hit). | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import sqlite3 | |
| import time | |
| import urllib.parse | |
| CACHE_SCHEMA_VERSION = 2 | |
| DEFAULT_TTL_S = 24 * 3600 | |
| DEFAULT_MAX_ENTRIES = 10_000 | |
| _COLUMNS = ["key", "url", "backend", "retrieved_at", "status", "http_status", | |
| "final_url", "title", "text", "language", "content_hash", | |
| "metadata", "links", "redirect_chain"] | |
| class WebCache: | |
| def __init__(self, path: str = ":memory:", ttl_s: float = DEFAULT_TTL_S, | |
| max_entries: int = DEFAULT_MAX_ENTRIES): | |
| self.ttl_s = ttl_s | |
| self.max_entries = max_entries | |
| self._db = sqlite3.connect(path) | |
| self._db.execute( | |
| "CREATE TABLE IF NOT EXISTS web_cache (" | |
| "key TEXT PRIMARY KEY, url TEXT, backend TEXT, " | |
| "retrieved_at REAL, status TEXT, http_status INTEGER, " | |
| "final_url TEXT, title TEXT, text TEXT, language TEXT, " | |
| "content_hash TEXT, metadata TEXT, links TEXT, redirect_chain TEXT)") | |
| self._db.commit() | |
| self._migrate() | |
| def _migrate(self): | |
| """Add columns introduced after a dev DB was created (ALTER TABLE | |
| ADD COLUMN is safe/idempotent behind a PRAGMA check).""" | |
| existing = {r[1] for r in self._db.execute("PRAGMA table_info(web_cache)")} | |
| for col in _COLUMNS: | |
| if col not in existing: | |
| self._db.execute( | |
| f"ALTER TABLE web_cache ADD COLUMN {col} TEXT DEFAULT ''") | |
| self._db.commit() | |
| # ------------------------------------------------------------------ # | |
| def canonicalize(url: str) -> str: | |
| p = urllib.parse.urlparse(url) | |
| scheme = p.scheme.lower() | |
| host = (p.hostname or "").lower() | |
| port = p.port | |
| netloc = host + (f":{port}" if port and port not in (80, 443) else "") | |
| path = p.path or "/" | |
| return urllib.parse.urlunparse((scheme, netloc, path, "", p.query, "")) | |
| def key(self, url: str, backend: str, backend_version: str, | |
| policy_version: str = "1", sanitizer_version: str = "1") -> str: | |
| """Cache identity: canonical URL + backend + versions.""" | |
| raw = "|".join([self.canonicalize(url), backend, backend_version, | |
| policy_version, sanitizer_version, | |
| f"schema{CACHE_SCHEMA_VERSION}"]) | |
| return hashlib.sha256(raw.encode("utf-8")).hexdigest() | |
| # ------------------------------------------------------------------ # | |
| def get(self, url: str, backend: str, backend_version: str, | |
| allow_stale: bool = False): | |
| """Return (entry_dict, fresh). Expired entries are returned ONLY with | |
| allow_stale=True and always carry stale=True + age metadata.""" | |
| row = self._db.execute( | |
| "SELECT url, backend, retrieved_at, status, http_status, " | |
| "final_url, title, text, language, content_hash, metadata, " | |
| "links, redirect_chain FROM web_cache WHERE key=?", | |
| (self.key(url, backend, backend_version),)).fetchone() | |
| if row is None: | |
| return None, False | |
| age = time.time() - row[2] | |
| fresh = age <= self.ttl_s | |
| if not fresh and not allow_stale: | |
| return None, False | |
| return {"requested_url": row[0], "backend": row[1], | |
| "retrieved_at": row[2], "status": row[3], | |
| "http_status": row[4], "final_url": row[5], "title": row[6], | |
| "text": row[7], "language": row[8] or None, | |
| "content_hash": row[9] or None, | |
| "metadata": json.loads(row[10] or "{}"), | |
| "links": json.loads(row[11] or "[]"), | |
| "redirect_chain": json.loads(row[12] or "[]"), | |
| "age_s": age, "stale": not fresh}, fresh | |
| def put(self, page, backend_version: str, metadata: dict | None = None) -> str: | |
| """Store a crawled page. Returns the content hash. Blocked/error | |
| pages are never persisted (H21-7).""" | |
| if getattr(page, "status", "ok") != "ok": | |
| return "" | |
| content_hash = hashlib.sha256( | |
| (getattr(page, "text", "") or "").encode("utf-8")).hexdigest() | |
| md = {"backend_version": backend_version, **(metadata or {})} | |
| self._db.execute( | |
| "INSERT OR REPLACE INTO web_cache (key,url,backend,retrieved_at," | |
| "status,http_status,final_url,title,text,language,content_hash," | |
| "metadata,links,redirect_chain) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", | |
| (self.key(page.requested_url, page.backend, backend_version), | |
| page.requested_url, page.backend, time.time(), | |
| getattr(page, "status", "ok"), page.http_status, page.final_url, | |
| page.title, page.text, getattr(page, "language", None), | |
| content_hash, json.dumps(md, ensure_ascii=False), | |
| json.dumps(page.links, ensure_ascii=False), | |
| json.dumps(page.redirect_chain, ensure_ascii=False))) | |
| self._db.commit() | |
| self._enforce_bound() | |
| return content_hash | |
| # ------------------------------------------------------------------ # | |
| def invalidate(self, url: str, backend: str | None = None, | |
| backend_version: str | None = None) -> int: | |
| """Explicit invalidation for one URL. backend given without a version | |
| removes ALL versions of that backend (H21-7 fix: the old 'version 0' | |
| key never matched stored entries, making invalidation a silent no-op).""" | |
| if backend is not None and backend_version is not None: | |
| cur = self._db.execute("DELETE FROM web_cache WHERE key=?", | |
| (self.key(url, backend, backend_version),)) | |
| elif backend is not None: | |
| cur = self._db.execute( | |
| "DELETE FROM web_cache WHERE url=? AND backend=?", | |
| (self.canonicalize(url), backend)) | |
| else: | |
| cur = self._db.execute("DELETE FROM web_cache WHERE url=?", | |
| (self.canonicalize(url),)) | |
| self._db.commit() | |
| return cur.rowcount | |
| def invalidate_domain(self, domain: str) -> int: | |
| cur = self._db.execute("DELETE FROM web_cache WHERE url LIKE ?", | |
| (f"%//{domain}/%",)) | |
| self._db.commit() | |
| return cur.rowcount | |
| def purge_expired(self) -> int: | |
| cutoff = time.time() - self.ttl_s | |
| cur = self._db.execute("DELETE FROM web_cache WHERE retrieved_at < ?", | |
| (cutoff,)) | |
| self._db.commit() | |
| return cur.rowcount | |
| def _enforce_bound(self): | |
| """Hard bound: evict oldest-retrieved entries beyond max_entries.""" | |
| n = self._db.execute("SELECT COUNT(*) FROM web_cache").fetchone()[0] | |
| if n > self.max_entries: | |
| excess = n - self.max_entries | |
| self._db.execute( | |
| "DELETE FROM web_cache WHERE key IN (SELECT key FROM web_cache " | |
| "ORDER BY retrieved_at ASC LIMIT ?)", (excess,)) | |
| self._db.commit() | |
| def close(self) -> None: | |
| """Release the SQLite connection (required on Windows before the | |
| underlying file can be deleted/moved).""" | |
| self._db.close() | |
| def stats(self) -> dict: | |
| n = self._db.execute("SELECT COUNT(*) FROM web_cache").fetchone()[0] | |
| return {"entries": n, "max_entries": self.max_entries, | |
| "ttl_s": self.ttl_s, "schema_version": CACHE_SCHEMA_VERSION} | |