dellweb3's picture
Deploy verified conversational workbench and isolated Java application previews
caf8059 verified
Raw History Blame Contribute Delete
6.12 kB
"""Audited read-only online resources; logical repositories can move to intranet."""
from __future__ import annotations
import hashlib
import json
import sqlite3
import time
from pathlib import Path
from urllib.error import HTTPError
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
DEFAULT_REPOSITORIES = {
"github-api": "https://api.github.com/",
"github-archive": "https://codeload.github.com/",
"github-raw": "https://raw.githubusercontent.com/",
"maven-central": "https://repo.maven.apache.org/maven2/",
}
class ResourceError(ValueError):
pass
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, *args):
return None
class ResourceClient:
def __init__(self, root: Path, repositories=None, opener=None):
self.root = Path(root)
self.root.mkdir(parents=True, exist_ok=True)
(self.root / "cache").mkdir(exist_ok=True)
self.repositories = dict(repositories or DEFAULT_REPOSITORIES)
self.opener = opener or build_opener(NoRedirect())
self.db = self.root / "resource-usage.sqlite"
with self.connect() as db:
db.execute("""CREATE TABLE IF NOT EXISTS resource_events(
id INTEGER PRIMARY KEY, timestamp REAL NOT NULL, repository TEXT NOT NULL,
logical_ref TEXT NOT NULL, url TEXT NOT NULL, outcome TEXT NOT NULL,
bytes_received INTEGER NOT NULL, content_bytes INTEGER NOT NULL,
sha256 TEXT, elapsed_ms INTEGER NOT NULL)""")
def connect(self):
db = sqlite3.connect(self.db, timeout=30)
db.row_factory = sqlite3.Row
return db
def resolve(self, repository: str, relative_path: str) -> str:
if repository not in self.repositories:
raise ResourceError("REPOSITORY_NOT_APPROVED")
base = self.repositories[repository]
parts = urlsplit(base)
if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment:
raise ResourceError("INVALID_REPOSITORY_ENDPOINT")
if any(x in relative_path for x in ("..", "\\", "?", "#", "%", ":")) or relative_path.startswith("/"):
raise ResourceError("INVALID_RESOURCE_PATH")
return base.rstrip("/") + "/" + relative_path
def record(self, repository, logical_ref, url, outcome, received, size, digest, started):
with self.connect() as db:
db.execute("INSERT INTO resource_events(timestamp,repository,logical_ref,url,outcome,bytes_received,content_bytes,sha256,elapsed_ms) VALUES(?,?,?,?,?,?,?,?,?)",
(time.time(), repository, logical_ref, url, outcome, received, size, digest,
round((time.monotonic() - started) * 1000)))
def fetch(self, repository, relative_path, *, logical_ref, max_bytes=512 * 1024 * 1024,
expected_sha256=None, refresh=False) -> Path:
url = self.resolve(repository, relative_path)
key = hashlib.sha256(url.encode()).hexdigest()
cache = self.root / "cache" / key
started = time.monotonic()
if cache.exists() and not refresh:
digest = hashlib.sha256(cache.read_bytes()).hexdigest()
if expected_sha256 and digest != expected_sha256:
self.record(repository, logical_ref, url, "cache_corrupt", 0, cache.stat().st_size, digest, started)
raise ResourceError("CACHE_DIGEST_MISMATCH")
self.record(repository, logical_ref, url, "cache_hit", 0, cache.stat().st_size, digest, started)
return cache
temporary = cache.with_suffix(".partial")
received = 0
try:
request = Request(url, headers={"User-Agent": "ZKCoding-Knowledge-Importer/1.0", "Accept": "application/vnd.github+json" if repository == "github-api" else "*/*"})
with self.opener.open(request, timeout=60) as response, temporary.open("wb") as output:
declared = response.headers.get("Content-Length")
if declared and int(declared) > max_bytes:
raise ResourceError("RESOURCE_TOO_LARGE")
while True:
chunk = response.read(min(1024 * 1024, max_bytes - received + 1))
if not chunk:
break
received += len(chunk)
if received > max_bytes:
raise ResourceError("RESOURCE_TOO_LARGE")
output.write(chunk)
digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
if expected_sha256 and digest != expected_sha256:
raise ResourceError("RESOURCE_DIGEST_MISMATCH")
temporary.replace(cache)
self.record(repository, logical_ref, url, "downloaded", received, received, digest, started)
return cache
except Exception as exc:
temporary.unlink(missing_ok=True)
self.record(repository, logical_ref, url, "http_" + str(exc.code) if isinstance(exc, HTTPError) else "failed", received, 0, None, started)
if isinstance(exc, ResourceError):
raise
raise ResourceError("RESOURCE_FETCH_FAILED") from None
def summary(self):
with self.connect() as db:
return [dict(r) for r in db.execute("""SELECT repository,logical_ref,url,
count(*) AS request_count, sum(CASE WHEN outcome='cache_hit' THEN 1 ELSE 0 END) AS cache_hits,
sum(CASE WHEN outcome='downloaded' THEN 1 ELSE 0 END) AS downloads,
sum(bytes_received) AS transferred_bytes, max(content_bytes) AS resource_bytes,
max(sha256) AS sha256 FROM resource_events GROUP BY repository,logical_ref,url ORDER BY repository,logical_ref""")]
def export(self, path: Path):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"schema": "zkcoding.resource-usage/v1", "resources": self.summary()}, ensure_ascii=False, indent=2), encoding="utf-8")