File size: 6,115 Bytes
caf8059
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
"""Audited read-only online resources; logical repositories can move to intranet."""
from __future__ import annotations

import hashlib
import json
import sqlite3
import time
from pathlib import Path
from urllib.error import HTTPError
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener


DEFAULT_REPOSITORIES = {
    "github-api": "https://api.github.com/",
    "github-archive": "https://codeload.github.com/",
    "github-raw": "https://raw.githubusercontent.com/",
    "maven-central": "https://repo.maven.apache.org/maven2/",
}


class ResourceError(ValueError):
    pass


class NoRedirect(HTTPRedirectHandler):
    def redirect_request(self, *args):
        return None


class ResourceClient:
    def __init__(self, root: Path, repositories=None, opener=None):
        self.root = Path(root)
        self.root.mkdir(parents=True, exist_ok=True)
        (self.root / "cache").mkdir(exist_ok=True)
        self.repositories = dict(repositories or DEFAULT_REPOSITORIES)
        self.opener = opener or build_opener(NoRedirect())
        self.db = self.root / "resource-usage.sqlite"
        with self.connect() as db:
            db.execute("""CREATE TABLE IF NOT EXISTS resource_events(
                id INTEGER PRIMARY KEY, timestamp REAL NOT NULL, repository TEXT NOT NULL,
                logical_ref TEXT NOT NULL, url TEXT NOT NULL, outcome TEXT NOT NULL,
                bytes_received INTEGER NOT NULL, content_bytes INTEGER NOT NULL,
                sha256 TEXT, elapsed_ms INTEGER NOT NULL)""")

    def connect(self):
        db = sqlite3.connect(self.db, timeout=30)
        db.row_factory = sqlite3.Row
        return db

    def resolve(self, repository: str, relative_path: str) -> str:
        if repository not in self.repositories:
            raise ResourceError("REPOSITORY_NOT_APPROVED")
        base = self.repositories[repository]
        parts = urlsplit(base)
        if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment:
            raise ResourceError("INVALID_REPOSITORY_ENDPOINT")
        if any(x in relative_path for x in ("..", "\\", "?", "#", "%", ":")) or relative_path.startswith("/"):
            raise ResourceError("INVALID_RESOURCE_PATH")
        return base.rstrip("/") + "/" + relative_path

    def record(self, repository, logical_ref, url, outcome, received, size, digest, started):
        with self.connect() as db:
            db.execute("INSERT INTO resource_events(timestamp,repository,logical_ref,url,outcome,bytes_received,content_bytes,sha256,elapsed_ms) VALUES(?,?,?,?,?,?,?,?,?)",
                       (time.time(), repository, logical_ref, url, outcome, received, size, digest,
                        round((time.monotonic() - started) * 1000)))

    def fetch(self, repository, relative_path, *, logical_ref, max_bytes=512 * 1024 * 1024,
              expected_sha256=None, refresh=False) -> Path:
        url = self.resolve(repository, relative_path)
        key = hashlib.sha256(url.encode()).hexdigest()
        cache = self.root / "cache" / key
        started = time.monotonic()
        if cache.exists() and not refresh:
            digest = hashlib.sha256(cache.read_bytes()).hexdigest()
            if expected_sha256 and digest != expected_sha256:
                self.record(repository, logical_ref, url, "cache_corrupt", 0, cache.stat().st_size, digest, started)
                raise ResourceError("CACHE_DIGEST_MISMATCH")
            self.record(repository, logical_ref, url, "cache_hit", 0, cache.stat().st_size, digest, started)
            return cache
        temporary = cache.with_suffix(".partial")
        received = 0
        try:
            request = Request(url, headers={"User-Agent": "ZKCoding-Knowledge-Importer/1.0", "Accept": "application/vnd.github+json" if repository == "github-api" else "*/*"})
            with self.opener.open(request, timeout=60) as response, temporary.open("wb") as output:
                declared = response.headers.get("Content-Length")
                if declared and int(declared) > max_bytes:
                    raise ResourceError("RESOURCE_TOO_LARGE")
                while True:
                    chunk = response.read(min(1024 * 1024, max_bytes - received + 1))
                    if not chunk:
                        break
                    received += len(chunk)
                    if received > max_bytes:
                        raise ResourceError("RESOURCE_TOO_LARGE")
                    output.write(chunk)
            digest = hashlib.sha256(temporary.read_bytes()).hexdigest()
            if expected_sha256 and digest != expected_sha256:
                raise ResourceError("RESOURCE_DIGEST_MISMATCH")
            temporary.replace(cache)
            self.record(repository, logical_ref, url, "downloaded", received, received, digest, started)
            return cache
        except Exception as exc:
            temporary.unlink(missing_ok=True)
            self.record(repository, logical_ref, url, "http_" + str(exc.code) if isinstance(exc, HTTPError) else "failed", received, 0, None, started)
            if isinstance(exc, ResourceError):
                raise
            raise ResourceError("RESOURCE_FETCH_FAILED") from None

    def summary(self):
        with self.connect() as db:
            return [dict(r) for r in db.execute("""SELECT repository,logical_ref,url,
                count(*) AS request_count, sum(CASE WHEN outcome='cache_hit' THEN 1 ELSE 0 END) AS cache_hits,
                sum(CASE WHEN outcome='downloaded' THEN 1 ELSE 0 END) AS downloads,
                sum(bytes_received) AS transferred_bytes, max(content_bytes) AS resource_bytes,
                max(sha256) AS sha256 FROM resource_events GROUP BY repository,logical_ref,url ORDER BY repository,logical_ref""")]

    def export(self, path: Path):
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(json.dumps({"schema": "zkcoding.resource-usage/v1", "resources": self.summary()}, ensure_ascii=False, indent=2), encoding="utf-8")