"""Audited read-only online resources; logical repositories can move to intranet.""" from __future__ import annotations import hashlib import json import sqlite3 import time from pathlib import Path from urllib.error import HTTPError from urllib.parse import urlsplit from urllib.request import HTTPRedirectHandler, Request, build_opener DEFAULT_REPOSITORIES = { "github-api": "https://api.github.com/", "github-archive": "https://codeload.github.com/", "github-raw": "https://raw.githubusercontent.com/", "maven-central": "https://repo.maven.apache.org/maven2/", } class ResourceError(ValueError): pass class NoRedirect(HTTPRedirectHandler): def redirect_request(self, *args): return None class ResourceClient: def __init__(self, root: Path, repositories=None, opener=None): self.root = Path(root) self.root.mkdir(parents=True, exist_ok=True) (self.root / "cache").mkdir(exist_ok=True) self.repositories = dict(repositories or DEFAULT_REPOSITORIES) self.opener = opener or build_opener(NoRedirect()) self.db = self.root / "resource-usage.sqlite" with self.connect() as db: db.execute("""CREATE TABLE IF NOT EXISTS resource_events( id INTEGER PRIMARY KEY, timestamp REAL NOT NULL, repository TEXT NOT NULL, logical_ref TEXT NOT NULL, url TEXT NOT NULL, outcome TEXT NOT NULL, bytes_received INTEGER NOT NULL, content_bytes INTEGER NOT NULL, sha256 TEXT, elapsed_ms INTEGER NOT NULL)""") def connect(self): db = sqlite3.connect(self.db, timeout=30) db.row_factory = sqlite3.Row return db def resolve(self, repository: str, relative_path: str) -> str: if repository not in self.repositories: raise ResourceError("REPOSITORY_NOT_APPROVED") base = self.repositories[repository] parts = urlsplit(base) if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment: raise ResourceError("INVALID_REPOSITORY_ENDPOINT") if any(x in relative_path for x in ("..", "\\", "?", "#", "%", ":")) or relative_path.startswith("/"): raise ResourceError("INVALID_RESOURCE_PATH") return base.rstrip("/") + "/" + relative_path def record(self, repository, logical_ref, url, outcome, received, size, digest, started): with self.connect() as db: db.execute("INSERT INTO resource_events(timestamp,repository,logical_ref,url,outcome,bytes_received,content_bytes,sha256,elapsed_ms) VALUES(?,?,?,?,?,?,?,?,?)", (time.time(), repository, logical_ref, url, outcome, received, size, digest, round((time.monotonic() - started) * 1000))) def fetch(self, repository, relative_path, *, logical_ref, max_bytes=512 * 1024 * 1024, expected_sha256=None, refresh=False) -> Path: url = self.resolve(repository, relative_path) key = hashlib.sha256(url.encode()).hexdigest() cache = self.root / "cache" / key started = time.monotonic() if cache.exists() and not refresh: digest = hashlib.sha256(cache.read_bytes()).hexdigest() if expected_sha256 and digest != expected_sha256: self.record(repository, logical_ref, url, "cache_corrupt", 0, cache.stat().st_size, digest, started) raise ResourceError("CACHE_DIGEST_MISMATCH") self.record(repository, logical_ref, url, "cache_hit", 0, cache.stat().st_size, digest, started) return cache temporary = cache.with_suffix(".partial") received = 0 try: request = Request(url, headers={"User-Agent": "ZKCoding-Knowledge-Importer/1.0", "Accept": "application/vnd.github+json" if repository == "github-api" else "*/*"}) with self.opener.open(request, timeout=60) as response, temporary.open("wb") as output: declared = response.headers.get("Content-Length") if declared and int(declared) > max_bytes: raise ResourceError("RESOURCE_TOO_LARGE") while True: chunk = response.read(min(1024 * 1024, max_bytes - received + 1)) if not chunk: break received += len(chunk) if received > max_bytes: raise ResourceError("RESOURCE_TOO_LARGE") output.write(chunk) digest = hashlib.sha256(temporary.read_bytes()).hexdigest() if expected_sha256 and digest != expected_sha256: raise ResourceError("RESOURCE_DIGEST_MISMATCH") temporary.replace(cache) self.record(repository, logical_ref, url, "downloaded", received, received, digest, started) return cache except Exception as exc: temporary.unlink(missing_ok=True) self.record(repository, logical_ref, url, "http_" + str(exc.code) if isinstance(exc, HTTPError) else "failed", received, 0, None, started) if isinstance(exc, ResourceError): raise raise ResourceError("RESOURCE_FETCH_FAILED") from None def summary(self): with self.connect() as db: return [dict(r) for r in db.execute("""SELECT repository,logical_ref,url, count(*) AS request_count, sum(CASE WHEN outcome='cache_hit' THEN 1 ELSE 0 END) AS cache_hits, sum(CASE WHEN outcome='downloaded' THEN 1 ELSE 0 END) AS downloads, sum(bytes_received) AS transferred_bytes, max(content_bytes) AS resource_bytes, max(sha256) AS sha256 FROM resource_events GROUP BY repository,logical_ref,url ORDER BY repository,logical_ref""")] def export(self, path: Path): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({"schema": "zkcoding.resource-usage/v1", "resources": self.summary()}, ensure_ascii=False, indent=2), encoding="utf-8")