Spaces:
Sleeping
Sleeping
Download src/adapters/artifact_store/resources.py from dellweb3/VibeCoding: direct link, hf CLI and curl.
- Browser
- Download file 6.12 kB
-
https://huggingface.co/spaces/dellweb3/VibeCoding/resolve/main/src/adapters/artifact_store/resources.py
- Command line
-
hf download hf://spaces/dellweb3/VibeCoding/src/adapters/artifact_store/resources.py
-
curl -L -o resources.py https://huggingface.co/spaces/dellweb3/VibeCoding/resolve/main/src/adapters/artifact_store/resources.py
6.12 kB
| """Audited read-only online resources; logical repositories can move to intranet.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import sqlite3 | |
| import time | |
| from pathlib import Path | |
| from urllib.error import HTTPError | |
| from urllib.parse import urlsplit | |
| from urllib.request import HTTPRedirectHandler, Request, build_opener | |
| DEFAULT_REPOSITORIES = { | |
| "github-api": "https://api.github.com/", | |
| "github-archive": "https://codeload.github.com/", | |
| "github-raw": "https://raw.githubusercontent.com/", | |
| "maven-central": "https://repo.maven.apache.org/maven2/", | |
| } | |
| class ResourceError(ValueError): | |
| pass | |
| class NoRedirect(HTTPRedirectHandler): | |
| def redirect_request(self, *args): | |
| return None | |
| class ResourceClient: | |
| def __init__(self, root: Path, repositories=None, opener=None): | |
| self.root = Path(root) | |
| self.root.mkdir(parents=True, exist_ok=True) | |
| (self.root / "cache").mkdir(exist_ok=True) | |
| self.repositories = dict(repositories or DEFAULT_REPOSITORIES) | |
| self.opener = opener or build_opener(NoRedirect()) | |
| self.db = self.root / "resource-usage.sqlite" | |
| with self.connect() as db: | |
| db.execute("""CREATE TABLE IF NOT EXISTS resource_events( | |
| id INTEGER PRIMARY KEY, timestamp REAL NOT NULL, repository TEXT NOT NULL, | |
| logical_ref TEXT NOT NULL, url TEXT NOT NULL, outcome TEXT NOT NULL, | |
| bytes_received INTEGER NOT NULL, content_bytes INTEGER NOT NULL, | |
| sha256 TEXT, elapsed_ms INTEGER NOT NULL)""") | |
| def connect(self): | |
| db = sqlite3.connect(self.db, timeout=30) | |
| db.row_factory = sqlite3.Row | |
| return db | |
| def resolve(self, repository: str, relative_path: str) -> str: | |
| if repository not in self.repositories: | |
| raise ResourceError("REPOSITORY_NOT_APPROVED") | |
| base = self.repositories[repository] | |
| parts = urlsplit(base) | |
| if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment: | |
| raise ResourceError("INVALID_REPOSITORY_ENDPOINT") | |
| if any(x in relative_path for x in ("..", "\\", "?", "#", "%", ":")) or relative_path.startswith("/"): | |
| raise ResourceError("INVALID_RESOURCE_PATH") | |
| return base.rstrip("/") + "/" + relative_path | |
| def record(self, repository, logical_ref, url, outcome, received, size, digest, started): | |
| with self.connect() as db: | |
| db.execute("INSERT INTO resource_events(timestamp,repository,logical_ref,url,outcome,bytes_received,content_bytes,sha256,elapsed_ms) VALUES(?,?,?,?,?,?,?,?,?)", | |
| (time.time(), repository, logical_ref, url, outcome, received, size, digest, | |
| round((time.monotonic() - started) * 1000))) | |
| def fetch(self, repository, relative_path, *, logical_ref, max_bytes=512 * 1024 * 1024, | |
| expected_sha256=None, refresh=False) -> Path: | |
| url = self.resolve(repository, relative_path) | |
| key = hashlib.sha256(url.encode()).hexdigest() | |
| cache = self.root / "cache" / key | |
| started = time.monotonic() | |
| if cache.exists() and not refresh: | |
| digest = hashlib.sha256(cache.read_bytes()).hexdigest() | |
| if expected_sha256 and digest != expected_sha256: | |
| self.record(repository, logical_ref, url, "cache_corrupt", 0, cache.stat().st_size, digest, started) | |
| raise ResourceError("CACHE_DIGEST_MISMATCH") | |
| self.record(repository, logical_ref, url, "cache_hit", 0, cache.stat().st_size, digest, started) | |
| return cache | |
| temporary = cache.with_suffix(".partial") | |
| received = 0 | |
| try: | |
| request = Request(url, headers={"User-Agent": "ZKCoding-Knowledge-Importer/1.0", "Accept": "application/vnd.github+json" if repository == "github-api" else "*/*"}) | |
| with self.opener.open(request, timeout=60) as response, temporary.open("wb") as output: | |
| declared = response.headers.get("Content-Length") | |
| if declared and int(declared) > max_bytes: | |
| raise ResourceError("RESOURCE_TOO_LARGE") | |
| while True: | |
| chunk = response.read(min(1024 * 1024, max_bytes - received + 1)) | |
| if not chunk: | |
| break | |
| received += len(chunk) | |
| if received > max_bytes: | |
| raise ResourceError("RESOURCE_TOO_LARGE") | |
| output.write(chunk) | |
| digest = hashlib.sha256(temporary.read_bytes()).hexdigest() | |
| if expected_sha256 and digest != expected_sha256: | |
| raise ResourceError("RESOURCE_DIGEST_MISMATCH") | |
| temporary.replace(cache) | |
| self.record(repository, logical_ref, url, "downloaded", received, received, digest, started) | |
| return cache | |
| except Exception as exc: | |
| temporary.unlink(missing_ok=True) | |
| self.record(repository, logical_ref, url, "http_" + str(exc.code) if isinstance(exc, HTTPError) else "failed", received, 0, None, started) | |
| if isinstance(exc, ResourceError): | |
| raise | |
| raise ResourceError("RESOURCE_FETCH_FAILED") from None | |
| def summary(self): | |
| with self.connect() as db: | |
| return [dict(r) for r in db.execute("""SELECT repository,logical_ref,url, | |
| count(*) AS request_count, sum(CASE WHEN outcome='cache_hit' THEN 1 ELSE 0 END) AS cache_hits, | |
| sum(CASE WHEN outcome='downloaded' THEN 1 ELSE 0 END) AS downloads, | |
| sum(bytes_received) AS transferred_bytes, max(content_bytes) AS resource_bytes, | |
| max(sha256) AS sha256 FROM resource_events GROUP BY repository,logical_ref,url ORDER BY repository,logical_ref""")] | |
| def export(self, path: Path): | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps({"schema": "zkcoding.resource-usage/v1", "resources": self.summary()}, ensure_ascii=False, indent=2), encoding="utf-8") | |