Spaces:
Running
Running
| """Pinned public Hugging Face sources, verified before decoding, bounded in memory.""" | |
| import hashlib | |
| import json | |
| import threading | |
| import urllib.error | |
| import urllib.request | |
| from collections import OrderedDict | |
| from pathlib import Path | |
| from urllib.parse import quote | |
| from tree_decoder import DecodeError, Document | |
| BASE = Path(__file__).resolve().parent | |
| DEFAULT_ID = "25519a9782d5d013" | |
| MAX_BYTES = 64 * 1024 * 1024 | |
| FEATURED = [DEFAULT_ID, "a707d971f504fc36", "ef3275fe2f08cfe4", "b0fa573b7a7681e3"] | |
| def source_url(row, action="resolve"): | |
| return "https://huggingface.co/datasets/%s/%s/%s/%s" % ( | |
| row["repo"], action, row["revision"], quote(row["path"], safe="/")) | |
| def verify_bytes(raw, row): | |
| if len(raw) != row["bytes"] or len(raw) > MAX_BYTES: | |
| raise DecodeError("Source size differs from the pinned catalog.") | |
| sha = hashlib.sha256(raw).hexdigest() | |
| if row.get("sha256"): | |
| matched = sha == row["sha256"] | |
| else: | |
| matched = hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest() == row["blob_id"] | |
| if not matched: | |
| raise DecodeError("Source hash differs from the pinned catalog. Decoding stopped.") | |
| return sha | |
| class SourceStore: | |
| def __init__(self, root=BASE, max_cached=3): | |
| self.root = Path(root) | |
| self.rows = json.loads((self.root / "catalog.json").read_text()) | |
| self.by_id = {row["id"]: row for row in self.rows} | |
| self.max_cached = max_cached | |
| self.cache = OrderedDict() | |
| self.lock = threading.Lock() | |
| def catalog(self): | |
| return {"files": self.rows, "default_id": DEFAULT_ID, "featured": FEATURED, | |
| "count": len(self.rows), "source_repositories": sorted({r["repo"] for r in self.rows}), | |
| "note": "Catalog includes conversation trees, linear chats and derived training files. " | |
| "The decoder identifies each file when opened."} | |
| def get(self, ident): | |
| if ident not in self.by_id: | |
| raise KeyError("Unknown catalog source.") | |
| with self.lock: | |
| if ident in self.cache: | |
| self.cache.move_to_end(ident) | |
| return self.cache[ident] | |
| row = self.by_id[ident] | |
| if row["bytes"] > MAX_BYTES: | |
| raise DecodeError("This source exceeds the demo's 64 MiB file limit. Use the Python decoder locally.") | |
| bundled = self.root / "starter" / (ident + ".json") | |
| if bundled.is_file(): | |
| raw = bundled.read_bytes() | |
| else: | |
| request = urllib.request.Request(source_url(row), headers={"User-Agent": "SML-Tree-Explorer/1.0"}) | |
| with urllib.request.urlopen(request, timeout=50) as response: | |
| raw = response.read(MAX_BYTES + 1) | |
| sha = verify_bytes(raw, row) | |
| document = Document(json.loads(raw)) | |
| provenance = {**row, "verified_sha256": sha, "verified": True, | |
| "source_url": source_url(row, "blob"), "download_url": source_url(row)} | |
| self.cache[ident] = (document, provenance) | |
| while len(self.cache) > self.max_cached: | |
| self.cache.popitem(last=False) | |
| return document, provenance | |