Buckets:
| import dataclasses | |
| import functools | |
| import os | |
| from abc import abstractmethod | |
| from collections.abc import Mapping | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING, Dict, Iterator, Literal, Optional, Union | |
| from urllib.parse import urlparse, urlunparse | |
| import sqlalchemy.pool | |
| from .utils import ensure_uri, path_from_uri, sanitize_uri | |
| if TYPE_CHECKING: | |
| import adbc_driver_manager.dbapi | |
| from obstore.store import AzureStore, GCSStore, LocalStore, S3Store | |
| __all__ = [ | |
| "DirectoryContainer", | |
| "EmbeddedSQLStorage", | |
| "RemoteSQLStorage", | |
| "FileStorage", | |
| "SQLStorage", | |
| "ObjectStorage", | |
| "Storage", | |
| "get_storage", | |
| "parse_storage", | |
| "unregister_storage", | |
| ] | |
| SUPPORTED_OBJECT_URI_SCHEMES = {"http", "https"} # TODO: Add "s3", "gs", "azure", "az" | |
| class Storage: | |
| "Base class for representing storage location" | |
| uri: str | |
| def __post_init__(self): | |
| object.__setattr__(self, "uri", ensure_uri(self.uri)) | |
| class FileStorage(Storage): | |
| "Filesystem storage location" | |
| def path(self): | |
| return path_from_uri(self.uri) | |
| def get_obstore_location(self, uri=None) -> "LocalStore": | |
| """Get an obstore.store.LocalStore instance rooted at specified URI. | |
| Parameters | |
| ---------- | |
| uri: str, optional | |
| """ | |
| if (uri is not None) and (not uri.startswith(self.uri)): | |
| raise ValueError( | |
| f"Requested URI {uri} is not within the base FileStorage URI {self.uri}" | |
| ) | |
| from obstore.store import LocalStore | |
| directory = path_from_uri(uri) | |
| directory.mkdir(parents=True, exist_ok=True) | |
| return LocalStore(directory) | |
| def _ensure_writable_location(uri: str) -> Path: | |
| "Ensure path is writable to avoid a confusing error message from driver." | |
| filepath = path_from_uri(uri) | |
| directory = Path(filepath).parent | |
| if directory.exists(): | |
| if not os.access(directory, os.X_OK | os.W_OK): | |
| raise ValueError( | |
| f"The directory {directory} exists but is not writable and executable." | |
| ) | |
| if Path(filepath).is_file() and (not os.access(filepath, os.W_OK)): | |
| raise ValueError(f"The path {filepath} exists but is not writable.") | |
| else: | |
| raise ValueError(f"The directory {directory} does not exist.") | |
| return filepath | |
| class ObjectStorage(Storage): | |
| """Bucket storage location for BLOBS | |
| The uri should include the bucket, but not the prefix within the bucket. This | |
| allows multiple ObjectStorage to point to different buckets with different | |
| credentials. | |
| Parameters | |
| ---------- | |
| uri: str | |
| Base URI, including bucket, but without prefix | |
| provider: Literal["s3", "azure", "google"] | |
| bucket: Optional[str] | |
| Only required for s3 and google | |
| username: Optional[str] | |
| password: Optional[str] | |
| config: dict | |
| Additional configuration options passed to obstore store classes. | |
| """ | |
| uri: str | |
| provider: Literal["s3", "azure", "google"] | |
| bucket: Optional[str] = None | |
| username: Optional[str] = None | |
| password: Optional[str] = None | |
| config: dict = dataclasses.field(default_factory=dict) | |
| def __post_init__(self): | |
| base_uri, bucket_name, _ = self.parse_blob_uri(ensure_uri(self.uri)) | |
| base_uri, username, password = sanitize_uri(base_uri) | |
| if (username is not None) or (password is not None): | |
| if ( | |
| (self.username is not None) | |
| or (self.password is not None) | |
| or ("username" in self.config) | |
| or ("password" in self.config) | |
| ): | |
| raise ValueError( | |
| "Credentials passed in URI and in username, password, or config fields." | |
| ) | |
| object.__setattr__(self, "username", username) | |
| object.__setattr__(self, "password", password) | |
| object.__setattr__(self, "uri", base_uri) | |
| object.__setattr__(self, "bucket", bucket_name) | |
| def parse_blob_uri(cls, uri: str) -> tuple[str, str, str]: | |
| """Split a blob URI into base URI, bucket name (optionally), and the prefix. | |
| For example, given 'http://example.com/bucket_name/path/to/blob', | |
| return ('http://example.com', 'bucket_name', 'path/to/blob'). | |
| """ | |
| # TODO: THIS NEEDS MORE WORK TO HANDLE S3, GCS, AZURE DIFFERENCES PROPERLY | |
| # CURRENTLY ONLY HANDLES HTTP(S) STYLE URIS | |
| parsed_uri = urlparse(uri) | |
| full_path = parsed_uri.path # includes bucket and the rest | |
| bucket_name, *blob_path = full_path.strip("/").split("/", 1) | |
| base_uri = f"{parsed_uri.scheme}://{parsed_uri.netloc}/{bucket_name}" | |
| return base_uri, bucket_name or None, "/".join(blob_path) | |
| def get_obstore_location( | |
| self, uri=None | |
| ) -> Union["S3Store", "AzureStore", "GCSStore"]: | |
| """Get an obstore.store instance rooted at specified URI. | |
| Parameters | |
| ---------- | |
| uri: str, optional | |
| The URI to use as the root for the obstore location. If not specified, use the base URI of | |
| this ObjectStorage. | |
| Returns | |
| ------- | |
| An instance of obstore.store.S3Store, obstore.store.AzureStore, or obstore.store.GCSStore, | |
| depending on the provider. | |
| """ | |
| if (uri is not None) and (not uri.startswith(self.uri)): | |
| raise ValueError( | |
| f"Requested URI {uri} is not within the base ObjectStorage URI {self.uri}" | |
| ) | |
| from obstore.store import AzureStore, GCSStore, S3Store | |
| # Build kwargs for the specific store class based on provider | |
| if self.provider == "s3": | |
| kwargs = { | |
| "endpoint": self.uri.split(self.bucket, -1)[0], | |
| "bucket": self.bucket, | |
| } | |
| if self.username is not None: | |
| kwargs["access_key_id"] = self.username | |
| if self.password is not None: | |
| kwargs["secret_access_key"] = self.password | |
| elif self.provider == "azure": | |
| kwargs = {"endpoint": self.uri} | |
| if self.username is not None: | |
| kwargs["client_id"] = self.username | |
| if self.password is not None: | |
| kwargs["client_secret"] = self.password | |
| elif self.provider == "google": | |
| kwargs = {"url": self.uri.split(self.bucket, -1)[0], "bucket": self.bucket} | |
| if self.password is not None: | |
| kwargs["service_account_key"] = self.password | |
| _class = {"s3": S3Store, "azure": AzureStore, "google": GCSStore}[self.provider] | |
| prefix = uri[len(self.uri) :].lstrip("/") if uri else None # noqa: E203 | |
| return _class(**kwargs, **self.config, prefix=prefix) | |
| class SQLStorage(Storage): | |
| "General purpose SQL database storage with connection pooling" | |
| pool_size: int = 5 | |
| max_overflow: int = 10 | |
| def __post_init__(self): | |
| # Ensure pool_size and max_overflow are set to a default if not specified | |
| if self.dialect == "duckdb": | |
| # DuckDB does not support pooling, so we use StaticPool | |
| object.__setattr__(self, "pool_size", 1) | |
| object.__setattr__(self, "max_overflow", 0) | |
| else: | |
| if self.pool_size is None: | |
| object.__setattr__(self, "pool_size", 5) | |
| if self.max_overflow is None: | |
| object.__setattr__(self, "max_overflow", 10) | |
| super().__post_init__() | |
| def create_adbc_connection(self) -> "adbc_driver_manager.dbapi.Connection": | |
| "Create a connection to the database." | |
| raise NotImplementedError("Subclasses must implement this method.") | |
| def dialect(self) -> str: | |
| "The database dialect (e.g. 'postgresql', 'sqlite', or 'duckdb')." | |
| return urlparse(self.uri).scheme | |
| def _adbc_connection(self) -> "adbc_driver_manager.dbapi.Connection": | |
| "A persistent connection to the database." | |
| return self.create_adbc_connection() | |
| def _connection_pool(self) -> "sqlalchemy.pool.QueuePool": | |
| from .server.metrics import monitor_db_pool | |
| creator = self._adbc_connection.adbc_clone | |
| if (self.dialect == "duckdb") or (":memory:" in self.uri): | |
| pool = sqlalchemy.pool.StaticPool(creator) | |
| else: | |
| pool = sqlalchemy.pool.QueuePool( | |
| creator, pool_size=self.pool_size, max_overflow=self.max_overflow | |
| ) | |
| monitor_db_pool(pool, self.uri) | |
| return pool | |
| def connect(self) -> "adbc_driver_manager.dbapi.Connection": | |
| "Get a connection from the pool." | |
| return self._connection_pool.connect() | |
| def dispose(self) -> None: | |
| "Close all connections and dispose of the connection pool." | |
| self._connection_pool.dispose() | |
| self._adbc_connection.close() | |
| class EmbeddedSQLStorage(SQLStorage): | |
| "File-based SQL database storage location" | |
| def create_adbc_connection(self) -> "adbc_driver_manager.dbapi.Connection": | |
| filepath = _ensure_writable_location(self.uri) | |
| if self.uri.startswith("duckdb:"): | |
| import adbc_driver_duckdb.dbapi | |
| return adbc_driver_duckdb.dbapi.connect(str(filepath)) | |
| elif self.uri.startswith("sqlite:"): | |
| import adbc_driver_sqlite.dbapi | |
| return adbc_driver_sqlite.dbapi.connect(str(filepath)) | |
| else: | |
| raise ValueError( | |
| f"Unsupported URI scheme {self.uri}: use 'duckdb:' or 'sqlite:'" | |
| ) | |
| class RemoteSQLStorage(SQLStorage): | |
| "Authenticated server-based SQL database storage location" | |
| username: Optional[str] = None | |
| password: Optional[str] = None | |
| def __post_init__(self): | |
| # Ensure the URI is sanitized and credentials are stored separately | |
| if not self.uri.startswith("postgresql:"): | |
| raise ValueError(f"Unsupported URI scheme {self.uri}: use 'postgresql:'") | |
| uri, username, password = sanitize_uri(self.uri) | |
| if (username is not None) or (password is not None): | |
| if (self.username is not None) or (self.password is not None): | |
| raise ValueError( | |
| "Credentials passed both in URI and in username/password fields." | |
| ) | |
| object.__setattr__(self, "uri", uri) | |
| object.__setattr__(self, "username", username) | |
| object.__setattr__(self, "password", password) | |
| super().__post_init__() | |
| def __repr__(self): | |
| return f"SQLStorage(uri={self.uri!r}, username={self.username!r})" | |
| def authenticated_uri(self): | |
| if self.username is None and self.password is None: | |
| return self.uri | |
| parsed_uri = urlparse(self.uri) | |
| components = ( | |
| parsed_uri.scheme, | |
| f"{self.username}:{self.password}@{parsed_uri.netloc}", | |
| parsed_uri.path, | |
| parsed_uri.params, | |
| parsed_uri.query, | |
| parsed_uri.fragment, | |
| ) | |
| return urlunparse(components) | |
| def create_adbc_connection(self) -> "adbc_driver_manager.dbapi.Connection": | |
| import adbc_driver_postgresql.dbapi | |
| return adbc_driver_postgresql.dbapi.connect(self.authenticated_uri) | |
| def parse_storage( | |
| item: Union[Path, str, dict], | |
| *, | |
| pool_size: int = 5, | |
| max_overflow: int = 10, | |
| ) -> Storage: | |
| "Create a Storage object from a URI or Path." | |
| if isinstance(item, dict): | |
| result = ObjectStorage( | |
| uri=item["uri"], | |
| provider=item["provider"], | |
| config=item.get("config", {}), | |
| ) | |
| else: | |
| item = ensure_uri(item) | |
| scheme = urlparse(item).scheme | |
| if scheme == "file": | |
| result = FileStorage(item) | |
| elif scheme == "postgresql": | |
| result = RemoteSQLStorage( | |
| item, pool_size=pool_size, max_overflow=max_overflow | |
| ) | |
| elif scheme in {"sqlite", "duckdb"}: | |
| result = EmbeddedSQLStorage( | |
| item, pool_size=pool_size, max_overflow=max_overflow | |
| ) | |
| else: | |
| raise ValueError(f"writable_storage item {item} has unrecognized scheme") | |
| return result | |
| # This global registry enables looking up Storage via URI, primarily for the | |
| # purpose of obtaining credentials, which are not stored in the catalog | |
| # database. | |
| _STORAGE: Dict[str, Storage] = {} | |
| def register_storage(storage: Storage) -> None: | |
| "Stash Storage for later lookup by URI." | |
| _STORAGE[storage.uri] = storage | |
| def unregister_storage(storage: Storage) -> None: | |
| "Remove Storage from the registry (call after dispose to release all references)." | |
| _STORAGE.pop(storage.uri, None) | |
| def get_storage(uri: str) -> Storage: | |
| "Look up Storage by URI." | |
| if urlparse(uri).scheme in SUPPORTED_OBJECT_URI_SCHEMES: | |
| uri, _, _ = ObjectStorage.parse_blob_uri(uri) | |
| return _STORAGE[uri] | |
| def size_from_uri(data_uri: str) -> int: | |
| """Return the byte length of the asset at `data_uri`. | |
| Dispatches on URI scheme: `file://` URIs are stat'd on the local | |
| filesystem; object-store URIs (those in `SUPPORTED_OBJECT_URI_SCHEMES`) | |
| issue a HEAD via obstore. Raises `ValueError` for any other scheme. | |
| Underlying I/O errors (`FileNotFoundError`, network errors, etc.) | |
| propagate; callers that want best-effort behavior should catch them at | |
| the call site. | |
| """ | |
| scheme = urlparse(data_uri).scheme | |
| if scheme == "file": | |
| return path_from_uri(data_uri).stat().st_size | |
| if scheme in SUPPORTED_OBJECT_URI_SCHEMES: | |
| storage = get_storage(data_uri) | |
| _, _, path = ObjectStorage.parse_blob_uri(data_uri) | |
| store = storage.get_obstore_location() | |
| return int(store.head(path)["size"]) | |
| raise ValueError( | |
| f"Cannot stat URI with unsupported scheme {scheme!r}: {data_uri!r}" | |
| ) | |
| def mtime_from_uri(data_uri: str) -> float: | |
| """Return the last-modified time of the asset at `data_uri`. | |
| The time is a POSIX timestamp (seconds since the epoch). | |
| Dispatches on URI scheme: `file://` URIs are stat'd on the local | |
| filesystem; object-store URIs (those in `SUPPORTED_OBJECT_URI_SCHEMES`) | |
| issue a HEAD via obstore. Raises `ValueError` for any other scheme. | |
| Underlying I/O errors (`FileNotFoundError`, network errors, etc.) | |
| propagate; callers that want best-effort behavior should catch them at | |
| the call site. | |
| """ | |
| scheme = urlparse(data_uri).scheme | |
| if scheme == "file": | |
| return path_from_uri(data_uri).stat().st_mtime | |
| if scheme in SUPPORTED_OBJECT_URI_SCHEMES: | |
| storage = get_storage(data_uri) | |
| _, _, path = ObjectStorage.parse_blob_uri(data_uri) | |
| store = storage.get_obstore_location() | |
| return store.head(path)["last_modified"].timestamp() | |
| raise ValueError( | |
| f"Cannot stat URI with unsupported scheme {scheme!r}: {data_uri!r}" | |
| ) | |
| class DirectoryContainer(Mapping[str, bytes]): | |
| """A storage container for byte-arrays representing Awkward Array buffers | |
| Each buffer is stored as a separate file in a given directory, with the | |
| filename corresponding to the form key. | |
| """ | |
| def __init__(self, directory: Path): | |
| self.directory = directory | |
| def __getitem__(self, key: str) -> bytes: | |
| with open(self.directory / key, "rb") as file: | |
| return file.read() | |
| def __setitem__(self, key: str, value: bytes) -> None: | |
| with open(self.directory / key, "wb") as file: | |
| file.write(value) | |
| def __delitem__(self, key: str) -> None: | |
| (self.directory / key).unlink(missing_ok=True) | |
| def __iter__(self) -> Iterator[str]: | |
| yield from (p.name for p in self.directory.iterdir()) | |
| def __len__(self) -> int: | |
| return sum(1 for _ in self.__iter__()) | |
Xet Storage Details
- Size:
- 16.3 kB
- Xet hash:
- 31b841e3c6bc805ad5c31718350ee51d3ebe8b25582e3c0e2f582cbb5f15e659
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.