Buckets:

download
raw
17.9 kB
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union
import dask.dataframe
import numpy
import pandas
import pytest
import ragged
import sparse
from numpy.typing import NDArray
from pytest_mock import MockFixture
from tiled.access_control.access_policies import ALL_ACCESS
from tiled.access_control.protocols import AccessPolicy
from tiled.access_control.scopes import ALL_SCOPES
from tiled.adapters.protocols import (
ArrayAdapter,
AwkwardAdapter,
BaseAdapter,
BytesAdapter,
RaggedAdapter,
SparseAdapter,
TableAdapter,
)
from tiled.ndslice import NDSlice
from tiled.server.schemas import Principal, PrincipalType
from tiled.storage import Storage
from tiled.structures.array import ArrayStructure, BuiltinDtype
from tiled.structures.awkward import AwkwardStructure
from tiled.structures.bytes import BytesStructure
from tiled.structures.core import Spec, StructureFamily
from tiled.structures.ragged import RaggedStructure
from tiled.structures.sparse import COOStructure
from tiled.structures.table import TableStructure
from tiled.type_aliases import JSON, AccessBlob, AccessTags, Filters, Scopes
class CustomArrayAdapter:
structure_family: Literal[StructureFamily.array] = StructureFamily.array
supported_storage: Set[type[Storage]] = set()
def __init__(
self,
array: NDArray[Any],
structure: ArrayStructure,
metadata: Optional[JSON] = None,
specs: Optional[List[Spec]] = None,
) -> None:
self._array = array
self._structure = structure
self._metadata = metadata or {}
self._specs = specs or []
def structure(self) -> ArrayStructure:
return self._structure
def read(self, slice: NDSlice) -> NDArray[Any]:
return self._array
def read_block(self, block: Tuple[int, ...]) -> NDArray[Any]:
return self._array
def specs(self) -> List[Spec]:
return self._specs
def metadata(self) -> JSON:
return self._metadata
def arrayadapter_protocol_functions(
adapter: ArrayAdapter, slice: NDSlice, block: Tuple[int, ...]
) -> None:
adapter.structure()
adapter.read(slice)
adapter.read_block(block)
adapter.specs()
adapter.metadata()
def test_arrayadapter_protocol(mocker: MockFixture) -> None:
mock_call = mocker.patch.object(CustomArrayAdapter, "structure")
mock_call2 = mocker.patch.object(CustomArrayAdapter, "read")
mock_call3 = mocker.patch.object(CustomArrayAdapter, "read_block")
mock_call4 = mocker.patch.object(CustomArrayAdapter, "specs")
mock_call5 = mocker.patch.object(CustomArrayAdapter, "metadata")
structure = ArrayStructure(
data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("int32")),
shape=(2, 512, 512),
chunks=((1, 1), (512,), (512,)),
dims=("time", "x", "y"), # optional
)
array = numpy.random.rand(2, 512, 512)
metadata: JSON = {"foo": "bar"}
anyslice = NDSlice(1, 1, 1)
anyblock = NDSlice(1, 1, 1)
anyarrayadapter = CustomArrayAdapter(array, structure, metadata=metadata)
assert anyarrayadapter.structure_family == StructureFamily.array
arrayadapter_protocol_functions(anyarrayadapter, anyslice, anyblock)
mock_call.assert_called_once()
mock_call2.assert_called_once_with(anyslice)
mock_call3.assert_called_once_with(anyblock)
mock_call4.assert_called_once()
mock_call5.assert_called_once()
class CustomAwkwardAdapter:
structure_family: Literal[StructureFamily.awkward] = StructureFamily.awkward
supported_storage: Set[type[Storage]] = set()
def __init__(
self,
container: dict[str, bytes],
structure: AwkwardStructure,
metadata: Optional[JSON] = None,
specs: Optional[List[Spec]] = None,
) -> None:
self.container = container
self._metadata = metadata or {}
self._structure = structure
self._specs = list(specs or [])
def structure(self) -> AwkwardStructure:
return self._structure
def read(self) -> NDArray[Any]:
return numpy.random.rand(4)
def read_buffers(self, form_keys: Optional[List[str]] = None) -> Dict[str, Any]:
return {"a": 123}
def specs(self) -> List[Spec]:
return self._specs
def metadata(self) -> JSON:
return self._metadata
def awkwardadapter_protocol_functions(
adapter: AwkwardAdapter,
slice: NDSlice,
form_keys: Optional[List[str]],
container: dict[str, bytes],
) -> None:
adapter.structure()
adapter.read()
adapter.read_buffers(form_keys)
adapter.specs()
adapter.metadata()
def test_awkwardadapter_protocol(mocker: MockFixture) -> None:
mock_call = mocker.patch.object(CustomAwkwardAdapter, "structure")
mock_call2 = mocker.patch.object(CustomAwkwardAdapter, "read")
mock_call3 = mocker.patch.object(CustomAwkwardAdapter, "read_buffers")
mock_call4 = mocker.patch.object(CustomAwkwardAdapter, "specs")
mock_call5 = mocker.patch.object(CustomAwkwardAdapter, "metadata")
structure = AwkwardStructure(length=2, form={"a": "b"})
metadata: JSON = {"foo": "bar"}
anyslice = NDSlice(1, 1, 1)
container = {"a": b"data"}
form_keys = ["a", "b", "c"]
anyawkwardadapter = CustomAwkwardAdapter(container, structure, metadata=metadata)
assert anyawkwardadapter.structure_family == StructureFamily.awkward
awkwardadapter_protocol_functions(anyawkwardadapter, anyslice, form_keys, container)
mock_call.assert_called_once()
mock_call2.assert_called_once()
mock_call3.assert_called_once_with(form_keys)
mock_call4.assert_called_once()
mock_call5.assert_called_once()
class CustomBytesAdapter:
structure_family: Literal[StructureFamily.bytes] = StructureFamily.bytes
supported_storage: Set[type[Storage]] = set()
def __init__(
self,
structure: BytesStructure,
metadata: Optional[JSON] = None,
specs: Optional[List[Spec]] = None,
) -> None:
self._structure = structure
self._metadata = metadata or {}
self._specs = list(specs or [])
def structure(self) -> BytesStructure:
return self._structure
def specs(self) -> List[Spec]:
return self._specs
def metadata(self) -> JSON:
return self._metadata
def bytesadapter_protocol_functions(adapter: BytesAdapter) -> None:
adapter.structure()
adapter.specs()
adapter.metadata()
def test_bytesadapter_protocol(mocker: MockFixture) -> None:
mock_call = mocker.patch.object(CustomBytesAdapter, "structure")
mock_call2 = mocker.patch.object(CustomBytesAdapter, "specs")
mock_call3 = mocker.patch.object(CustomBytesAdapter, "metadata")
structure = BytesStructure()
metadata: JSON = {"foo": "bar"}
anybytesadapter = CustomBytesAdapter(structure, metadata=metadata)
assert anybytesadapter.structure_family == StructureFamily.bytes
bytesadapter_protocol_functions(anybytesadapter)
mock_call.assert_called_once()
mock_call2.assert_called_once()
mock_call3.assert_called_once()
class CustomRaggedAdapter:
structure_family: Literal[StructureFamily.ragged] = StructureFamily.ragged
supported_storage: Set[type[Storage]] = set()
def __init__(
self,
array: ragged.array,
structure: RaggedStructure,
metadata: Optional[JSON] = None,
specs: Optional[List[Spec]] = None,
) -> None:
self._array = array
self._structure = structure
self._metadata = metadata or {}
self._specs = list(specs or [])
def structure(self) -> RaggedStructure:
return self._structure
def read(self, slice: Optional[NDSlice] = None) -> ragged.array:
return self._array
def specs(self) -> List[Spec]:
return self._specs
def metadata(self) -> JSON:
return self._metadata
def raggedadapter_protocol_functions(
adapter: RaggedAdapter, slice: Optional[NDSlice]
) -> None:
adapter.structure()
adapter.read(slice)
adapter.specs()
adapter.metadata()
def test_raggedadapter_protocol(mocker: MockFixture) -> None:
mock_call = mocker.patch.object(CustomRaggedAdapter, "structure")
mock_call2 = mocker.patch.object(CustomRaggedAdapter, "read")
mock_call3 = mocker.patch.object(CustomRaggedAdapter, "specs")
mock_call4 = mocker.patch.object(CustomRaggedAdapter, "metadata")
structure = RaggedStructure(
data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("float64")),
shape=(2, None),
size=5,
chunks=((1, 1), None),
dims=("time", "x"),
)
array = ragged.array([[1.0, 2.0, 3.0], [4.0, 5.0]])
metadata: JSON = {"foo": "bar"}
anyslice = NDSlice(0)
anyraggedadapter = CustomRaggedAdapter(array, structure, metadata=metadata)
assert anyraggedadapter.structure_family == StructureFamily.ragged
raggedadapter_protocol_functions(anyraggedadapter, anyslice)
mock_call.assert_called_once()
mock_call2.assert_called_once_with(anyslice)
mock_call3.assert_called_once()
mock_call4.assert_called_once()
class CustomSparseAdapter:
structure_family: Literal[StructureFamily.sparse] = StructureFamily.sparse
supported_storage: Set[type[Storage]] = set()
def __init__(
self,
blocks: Dict[Tuple[int, ...], Tuple[NDArray[Any], Any]],
structure: COOStructure,
*,
metadata: Optional[JSON] = None,
specs: Optional[List[Spec]] = None,
) -> None:
"""
Construct from blocks with coords given in block-local reference frame.
Parameters
----------
blocks :
structure :
metadata :
specs :
"""
self.blocks = blocks
self._metadata = metadata or {}
self._structure = structure
self._specs = specs or []
all_coords = [[1], [1]]
all_data = [[1], [1]]
self.arr = sparse.COO(
data=numpy.concatenate(all_data),
coords=numpy.concatenate(all_coords, axis=-1),
shape=self._structure.shape,
)
def structure(self) -> COOStructure:
return self._structure
def read(self, slice: NDSlice) -> sparse.COO:
return self.arr
def read_block(self, block: Tuple[int, ...]) -> sparse.COO:
return self.arr
def specs(self) -> List[Spec]:
return self._specs
def metadata(self) -> JSON:
return self._metadata
def sparseadapter_protocol_functions(
adapter: SparseAdapter, slice: NDSlice, block: Tuple[int, ...]
) -> None:
adapter.structure()
adapter.read(slice)
adapter.read_block(block)
adapter.specs()
adapter.metadata()
def test_sparseadapter_protocol(mocker: MockFixture) -> None:
mock_call = mocker.patch.object(CustomSparseAdapter, "structure")
mock_call2 = mocker.patch.object(CustomSparseAdapter, "read")
mock_call3 = mocker.patch.object(CustomSparseAdapter, "read_block")
mock_call4 = mocker.patch.object(CustomSparseAdapter, "specs")
mock_call5 = mocker.patch.object(CustomSparseAdapter, "metadata")
array = numpy.random.rand(2, 512, 512)
structure = COOStructure(
shape=(2 * 5,),
chunks=((5, 5),),
data_type=BuiltinDtype.from_numpy_dtype(array.dtype),
)
blocks: Dict[Tuple[int, ...], Tuple[NDArray[Any], Any]] = {(1,): (array, (1,))}
metadata: JSON = {"foo": "bar"}
anyslice = NDSlice(1, 1, 1)
anyblock = NDSlice(1, 1, 1)
anysparseadapter = CustomSparseAdapter(blocks, structure, metadata=metadata)
assert anysparseadapter.structure_family == StructureFamily.sparse
sparseadapter_protocol_functions(anysparseadapter, anyslice, anyblock)
mock_call.assert_called_once()
mock_call2.assert_called_once_with(anyslice)
mock_call3.assert_called_once_with(anyblock)
mock_call4.assert_called_once()
mock_call5.assert_called_once()
class CustomTableAdapter:
structure_family: Literal[StructureFamily.table] = StructureFamily.table
supported_storage: Set[type[Storage]] = set()
def __init__(
self,
partitions: List[Any],
structure: TableStructure,
*,
metadata: Optional[JSON] = None,
specs: Optional[List[Spec]] = None,
) -> None:
"""
Parameters
----------
partitions :
structure :
metadata :
specs :
"""
self._metadata = metadata or {}
self._partitions = list(partitions)
self._structure = structure
self._specs = specs or []
def structure(self) -> TableStructure:
return self._structure
def read(
self, fields: Optional[List[str]] = None
) -> Union[dask.dataframe.DataFrame, pandas.DataFrame]:
return self._partitions
def read_partition(
self,
partition: int,
fields: Optional[str] = None,
) -> Union[dask.dataframe.DataFrame, pandas.DataFrame]:
return self._partitions[partition]
def specs(self) -> List[Spec]:
return self._specs
def metadata(self) -> JSON:
return self._metadata
def __getitem__(self, key: str) -> ArrayAdapter:
array = numpy.random.rand(2, 512, 512)
metadata: JSON = {"foo": "bar"}
structure = ArrayStructure(
data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("int32")),
shape=(2, 512, 512),
chunks=((1, 1), (512,), (512,)),
dims=("time", "x", "y"), # optional
)
return CustomArrayAdapter(array, structure, metadata=metadata)
def tableadapter_protocol_functions(
adapter: TableAdapter, partition: int, fields: List[str]
) -> None:
adapter.structure()
adapter.read(fields)
adapter.read_partition(partition)
adapter.specs()
adapter.metadata()
adapter["abc"]
def test_tableadapter_protocol(mocker: MockFixture) -> None:
mock_call = mocker.patch.object(CustomTableAdapter, "structure")
mock_call2 = mocker.patch.object(CustomTableAdapter, "read")
mock_call3 = mocker.patch.object(CustomTableAdapter, "read_partition")
mock_call4 = mocker.patch.object(CustomTableAdapter, "specs")
mock_call5 = mocker.patch.object(CustomTableAdapter, "metadata")
mock_call6 = mocker.patch.object(CustomTableAdapter, "__getitem__")
structure = TableStructure(
arrow_schema="a", npartitions=1, columns=["A"], resizable=False
)
partitions = pandas.DataFrame([1])
metadata: JSON = {"foo": "bar"}
fields = ["a", "b", "c"]
partition = 1
anytableadapter = CustomTableAdapter(partitions, structure, metadata=metadata)
assert anytableadapter.structure_family == StructureFamily.table
tableadapter_protocol_functions(anytableadapter, partition, fields)
mock_call.assert_called_once()
mock_call2.assert_called_once_with(fields)
mock_call3.assert_called_once_with(partition)
mock_call4.assert_called_once()
mock_call5.assert_called_once()
mock_call6.assert_called_once_with("abc")
class CustomAccessPolicy(AccessPolicy):
def __init__(self, scopes: Optional[Scopes] = None) -> None:
self.scopes = scopes if (scopes is not None) else ALL_SCOPES
def _get_id(self, principal: Principal) -> None:
return None
async def init_node(
self,
principal: Principal,
authn_access_tags: Optional[AccessTags],
authn_scopes: Scopes,
access_blob: Optional[AccessBlob] = None,
) -> Tuple[bool, Optional[AccessBlob]]:
return (False, access_blob)
async def modify_node(
self,
node: BaseAdapter,
principal: Principal,
authn_access_tags: Optional[AccessTags],
authn_scopes: Scopes,
access_blob: Optional[AccessBlob] = None,
) -> Tuple[bool, Optional[AccessBlob]]:
return (False, access_blob)
async def allowed_scopes(
self,
node: BaseAdapter,
principal: Principal,
authn_access_tags: Optional[AccessTags],
authn_scopes: Scopes,
) -> Scopes:
allowed = self.scopes
somemetadata = node.metadata() # noqa: F841
return allowed
async def filters(
self,
node: BaseAdapter,
principal: Principal,
authn_access_tags: Optional[AccessTags],
authn_scopes: Scopes,
scopes: Scopes,
) -> Filters:
queries: Filters = ALL_ACCESS
somespecs = node.specs() # noqa: F841
return queries
async def accesspolicy_protocol_functions(
policy: AccessPolicy,
node: BaseAdapter,
principal: Principal,
authn_access_tags: Optional[AccessTags],
authn_scopes: Scopes,
scopes: Scopes,
) -> None:
await policy.allowed_scopes(node, principal, authn_access_tags, authn_scopes)
await policy.filters(node, principal, authn_access_tags, authn_scopes, scopes)
@pytest.mark.asyncio # type: ignore
async def test_accesspolicy_protocol(mocker: MockFixture) -> None:
mock_call = mocker.patch.object(CustomAwkwardAdapter, "metadata")
mock_call2 = mocker.patch.object(CustomAwkwardAdapter, "specs")
anyaccesspolicy = CustomAccessPolicy(scopes={"a12mdjnk4"})
structure = AwkwardStructure(length=2, form={"a": "b"})
metadata: JSON = {"foo": "bar"}
container = {"a": b"data"}
principal = Principal(
uuid="12345678124123412345678123456781", type=PrincipalType.user
)
authn_access_tags = {"qux", "quux"}
authn_scopes = {"abc", "baz"}
scopes = {"abc"}
anyawkwardadapter = CustomAwkwardAdapter(container, structure, metadata=metadata)
await accesspolicy_protocol_functions(
anyaccesspolicy,
anyawkwardadapter,
principal,
authn_access_tags,
authn_scopes,
scopes,
)
mock_call.assert_called_once()
mock_call2.assert_called_once()

Xet Storage Details

Size:
17.9 kB
·
Xet hash:
b090251a2aa779dd77cceecdc465a5dacf78ffa7fa9f35226ec226e9fe3934f5

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.