Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /bluesky_tiled /tests /test_protocols.py
| from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union | |
| import dask.dataframe | |
| import numpy | |
| import pandas | |
| import pytest | |
| import ragged | |
| import sparse | |
| from numpy.typing import NDArray | |
| from pytest_mock import MockFixture | |
| from tiled.access_control.access_policies import ALL_ACCESS | |
| from tiled.access_control.protocols import AccessPolicy | |
| from tiled.access_control.scopes import ALL_SCOPES | |
| from tiled.adapters.protocols import ( | |
| ArrayAdapter, | |
| AwkwardAdapter, | |
| BaseAdapter, | |
| BytesAdapter, | |
| RaggedAdapter, | |
| SparseAdapter, | |
| TableAdapter, | |
| ) | |
| from tiled.ndslice import NDSlice | |
| from tiled.server.schemas import Principal, PrincipalType | |
| from tiled.storage import Storage | |
| from tiled.structures.array import ArrayStructure, BuiltinDtype | |
| from tiled.structures.awkward import AwkwardStructure | |
| from tiled.structures.bytes import BytesStructure | |
| from tiled.structures.core import Spec, StructureFamily | |
| from tiled.structures.ragged import RaggedStructure | |
| from tiled.structures.sparse import COOStructure | |
| from tiled.structures.table import TableStructure | |
| from tiled.type_aliases import JSON, AccessBlob, AccessTags, Filters, Scopes | |
| class CustomArrayAdapter: | |
| structure_family: Literal[StructureFamily.array] = StructureFamily.array | |
| supported_storage: Set[type[Storage]] = set() | |
| def __init__( | |
| self, | |
| array: NDArray[Any], | |
| structure: ArrayStructure, | |
| metadata: Optional[JSON] = None, | |
| specs: Optional[List[Spec]] = None, | |
| ) -> None: | |
| self._array = array | |
| self._structure = structure | |
| self._metadata = metadata or {} | |
| self._specs = specs or [] | |
| def structure(self) -> ArrayStructure: | |
| return self._structure | |
| def read(self, slice: NDSlice) -> NDArray[Any]: | |
| return self._array | |
| def read_block(self, block: Tuple[int, ...]) -> NDArray[Any]: | |
| return self._array | |
| def specs(self) -> List[Spec]: | |
| return self._specs | |
| def metadata(self) -> JSON: | |
| return self._metadata | |
| def arrayadapter_protocol_functions( | |
| adapter: ArrayAdapter, slice: NDSlice, block: Tuple[int, ...] | |
| ) -> None: | |
| adapter.structure() | |
| adapter.read(slice) | |
| adapter.read_block(block) | |
| adapter.specs() | |
| adapter.metadata() | |
| def test_arrayadapter_protocol(mocker: MockFixture) -> None: | |
| mock_call = mocker.patch.object(CustomArrayAdapter, "structure") | |
| mock_call2 = mocker.patch.object(CustomArrayAdapter, "read") | |
| mock_call3 = mocker.patch.object(CustomArrayAdapter, "read_block") | |
| mock_call4 = mocker.patch.object(CustomArrayAdapter, "specs") | |
| mock_call5 = mocker.patch.object(CustomArrayAdapter, "metadata") | |
| structure = ArrayStructure( | |
| data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("int32")), | |
| shape=(2, 512, 512), | |
| chunks=((1, 1), (512,), (512,)), | |
| dims=("time", "x", "y"), # optional | |
| ) | |
| array = numpy.random.rand(2, 512, 512) | |
| metadata: JSON = {"foo": "bar"} | |
| anyslice = NDSlice(1, 1, 1) | |
| anyblock = NDSlice(1, 1, 1) | |
| anyarrayadapter = CustomArrayAdapter(array, structure, metadata=metadata) | |
| assert anyarrayadapter.structure_family == StructureFamily.array | |
| arrayadapter_protocol_functions(anyarrayadapter, anyslice, anyblock) | |
| mock_call.assert_called_once() | |
| mock_call2.assert_called_once_with(anyslice) | |
| mock_call3.assert_called_once_with(anyblock) | |
| mock_call4.assert_called_once() | |
| mock_call5.assert_called_once() | |
| class CustomAwkwardAdapter: | |
| structure_family: Literal[StructureFamily.awkward] = StructureFamily.awkward | |
| supported_storage: Set[type[Storage]] = set() | |
| def __init__( | |
| self, | |
| container: dict[str, bytes], | |
| structure: AwkwardStructure, | |
| metadata: Optional[JSON] = None, | |
| specs: Optional[List[Spec]] = None, | |
| ) -> None: | |
| self.container = container | |
| self._metadata = metadata or {} | |
| self._structure = structure | |
| self._specs = list(specs or []) | |
| def structure(self) -> AwkwardStructure: | |
| return self._structure | |
| def read(self) -> NDArray[Any]: | |
| return numpy.random.rand(4) | |
| def read_buffers(self, form_keys: Optional[List[str]] = None) -> Dict[str, Any]: | |
| return {"a": 123} | |
| def specs(self) -> List[Spec]: | |
| return self._specs | |
| def metadata(self) -> JSON: | |
| return self._metadata | |
| def awkwardadapter_protocol_functions( | |
| adapter: AwkwardAdapter, | |
| slice: NDSlice, | |
| form_keys: Optional[List[str]], | |
| container: dict[str, bytes], | |
| ) -> None: | |
| adapter.structure() | |
| adapter.read() | |
| adapter.read_buffers(form_keys) | |
| adapter.specs() | |
| adapter.metadata() | |
| def test_awkwardadapter_protocol(mocker: MockFixture) -> None: | |
| mock_call = mocker.patch.object(CustomAwkwardAdapter, "structure") | |
| mock_call2 = mocker.patch.object(CustomAwkwardAdapter, "read") | |
| mock_call3 = mocker.patch.object(CustomAwkwardAdapter, "read_buffers") | |
| mock_call4 = mocker.patch.object(CustomAwkwardAdapter, "specs") | |
| mock_call5 = mocker.patch.object(CustomAwkwardAdapter, "metadata") | |
| structure = AwkwardStructure(length=2, form={"a": "b"}) | |
| metadata: JSON = {"foo": "bar"} | |
| anyslice = NDSlice(1, 1, 1) | |
| container = {"a": b"data"} | |
| form_keys = ["a", "b", "c"] | |
| anyawkwardadapter = CustomAwkwardAdapter(container, structure, metadata=metadata) | |
| assert anyawkwardadapter.structure_family == StructureFamily.awkward | |
| awkwardadapter_protocol_functions(anyawkwardadapter, anyslice, form_keys, container) | |
| mock_call.assert_called_once() | |
| mock_call2.assert_called_once() | |
| mock_call3.assert_called_once_with(form_keys) | |
| mock_call4.assert_called_once() | |
| mock_call5.assert_called_once() | |
| class CustomBytesAdapter: | |
| structure_family: Literal[StructureFamily.bytes] = StructureFamily.bytes | |
| supported_storage: Set[type[Storage]] = set() | |
| def __init__( | |
| self, | |
| structure: BytesStructure, | |
| metadata: Optional[JSON] = None, | |
| specs: Optional[List[Spec]] = None, | |
| ) -> None: | |
| self._structure = structure | |
| self._metadata = metadata or {} | |
| self._specs = list(specs or []) | |
| def structure(self) -> BytesStructure: | |
| return self._structure | |
| def specs(self) -> List[Spec]: | |
| return self._specs | |
| def metadata(self) -> JSON: | |
| return self._metadata | |
| def bytesadapter_protocol_functions(adapter: BytesAdapter) -> None: | |
| adapter.structure() | |
| adapter.specs() | |
| adapter.metadata() | |
| def test_bytesadapter_protocol(mocker: MockFixture) -> None: | |
| mock_call = mocker.patch.object(CustomBytesAdapter, "structure") | |
| mock_call2 = mocker.patch.object(CustomBytesAdapter, "specs") | |
| mock_call3 = mocker.patch.object(CustomBytesAdapter, "metadata") | |
| structure = BytesStructure() | |
| metadata: JSON = {"foo": "bar"} | |
| anybytesadapter = CustomBytesAdapter(structure, metadata=metadata) | |
| assert anybytesadapter.structure_family == StructureFamily.bytes | |
| bytesadapter_protocol_functions(anybytesadapter) | |
| mock_call.assert_called_once() | |
| mock_call2.assert_called_once() | |
| mock_call3.assert_called_once() | |
| class CustomRaggedAdapter: | |
| structure_family: Literal[StructureFamily.ragged] = StructureFamily.ragged | |
| supported_storage: Set[type[Storage]] = set() | |
| def __init__( | |
| self, | |
| array: ragged.array, | |
| structure: RaggedStructure, | |
| metadata: Optional[JSON] = None, | |
| specs: Optional[List[Spec]] = None, | |
| ) -> None: | |
| self._array = array | |
| self._structure = structure | |
| self._metadata = metadata or {} | |
| self._specs = list(specs or []) | |
| def structure(self) -> RaggedStructure: | |
| return self._structure | |
| def read(self, slice: Optional[NDSlice] = None) -> ragged.array: | |
| return self._array | |
| def specs(self) -> List[Spec]: | |
| return self._specs | |
| def metadata(self) -> JSON: | |
| return self._metadata | |
| def raggedadapter_protocol_functions( | |
| adapter: RaggedAdapter, slice: Optional[NDSlice] | |
| ) -> None: | |
| adapter.structure() | |
| adapter.read(slice) | |
| adapter.specs() | |
| adapter.metadata() | |
| def test_raggedadapter_protocol(mocker: MockFixture) -> None: | |
| mock_call = mocker.patch.object(CustomRaggedAdapter, "structure") | |
| mock_call2 = mocker.patch.object(CustomRaggedAdapter, "read") | |
| mock_call3 = mocker.patch.object(CustomRaggedAdapter, "specs") | |
| mock_call4 = mocker.patch.object(CustomRaggedAdapter, "metadata") | |
| structure = RaggedStructure( | |
| data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("float64")), | |
| shape=(2, None), | |
| size=5, | |
| chunks=((1, 1), None), | |
| dims=("time", "x"), | |
| ) | |
| array = ragged.array([[1.0, 2.0, 3.0], [4.0, 5.0]]) | |
| metadata: JSON = {"foo": "bar"} | |
| anyslice = NDSlice(0) | |
| anyraggedadapter = CustomRaggedAdapter(array, structure, metadata=metadata) | |
| assert anyraggedadapter.structure_family == StructureFamily.ragged | |
| raggedadapter_protocol_functions(anyraggedadapter, anyslice) | |
| mock_call.assert_called_once() | |
| mock_call2.assert_called_once_with(anyslice) | |
| mock_call3.assert_called_once() | |
| mock_call4.assert_called_once() | |
| class CustomSparseAdapter: | |
| structure_family: Literal[StructureFamily.sparse] = StructureFamily.sparse | |
| supported_storage: Set[type[Storage]] = set() | |
| def __init__( | |
| self, | |
| blocks: Dict[Tuple[int, ...], Tuple[NDArray[Any], Any]], | |
| structure: COOStructure, | |
| *, | |
| metadata: Optional[JSON] = None, | |
| specs: Optional[List[Spec]] = None, | |
| ) -> None: | |
| """ | |
| Construct from blocks with coords given in block-local reference frame. | |
| Parameters | |
| ---------- | |
| blocks : | |
| structure : | |
| metadata : | |
| specs : | |
| """ | |
| self.blocks = blocks | |
| self._metadata = metadata or {} | |
| self._structure = structure | |
| self._specs = specs or [] | |
| all_coords = [[1], [1]] | |
| all_data = [[1], [1]] | |
| self.arr = sparse.COO( | |
| data=numpy.concatenate(all_data), | |
| coords=numpy.concatenate(all_coords, axis=-1), | |
| shape=self._structure.shape, | |
| ) | |
| def structure(self) -> COOStructure: | |
| return self._structure | |
| def read(self, slice: NDSlice) -> sparse.COO: | |
| return self.arr | |
| def read_block(self, block: Tuple[int, ...]) -> sparse.COO: | |
| return self.arr | |
| def specs(self) -> List[Spec]: | |
| return self._specs | |
| def metadata(self) -> JSON: | |
| return self._metadata | |
| def sparseadapter_protocol_functions( | |
| adapter: SparseAdapter, slice: NDSlice, block: Tuple[int, ...] | |
| ) -> None: | |
| adapter.structure() | |
| adapter.read(slice) | |
| adapter.read_block(block) | |
| adapter.specs() | |
| adapter.metadata() | |
| def test_sparseadapter_protocol(mocker: MockFixture) -> None: | |
| mock_call = mocker.patch.object(CustomSparseAdapter, "structure") | |
| mock_call2 = mocker.patch.object(CustomSparseAdapter, "read") | |
| mock_call3 = mocker.patch.object(CustomSparseAdapter, "read_block") | |
| mock_call4 = mocker.patch.object(CustomSparseAdapter, "specs") | |
| mock_call5 = mocker.patch.object(CustomSparseAdapter, "metadata") | |
| array = numpy.random.rand(2, 512, 512) | |
| structure = COOStructure( | |
| shape=(2 * 5,), | |
| chunks=((5, 5),), | |
| data_type=BuiltinDtype.from_numpy_dtype(array.dtype), | |
| ) | |
| blocks: Dict[Tuple[int, ...], Tuple[NDArray[Any], Any]] = {(1,): (array, (1,))} | |
| metadata: JSON = {"foo": "bar"} | |
| anyslice = NDSlice(1, 1, 1) | |
| anyblock = NDSlice(1, 1, 1) | |
| anysparseadapter = CustomSparseAdapter(blocks, structure, metadata=metadata) | |
| assert anysparseadapter.structure_family == StructureFamily.sparse | |
| sparseadapter_protocol_functions(anysparseadapter, anyslice, anyblock) | |
| mock_call.assert_called_once() | |
| mock_call2.assert_called_once_with(anyslice) | |
| mock_call3.assert_called_once_with(anyblock) | |
| mock_call4.assert_called_once() | |
| mock_call5.assert_called_once() | |
| class CustomTableAdapter: | |
| structure_family: Literal[StructureFamily.table] = StructureFamily.table | |
| supported_storage: Set[type[Storage]] = set() | |
| def __init__( | |
| self, | |
| partitions: List[Any], | |
| structure: TableStructure, | |
| *, | |
| metadata: Optional[JSON] = None, | |
| specs: Optional[List[Spec]] = None, | |
| ) -> None: | |
| """ | |
| Parameters | |
| ---------- | |
| partitions : | |
| structure : | |
| metadata : | |
| specs : | |
| """ | |
| self._metadata = metadata or {} | |
| self._partitions = list(partitions) | |
| self._structure = structure | |
| self._specs = specs or [] | |
| def structure(self) -> TableStructure: | |
| return self._structure | |
| def read( | |
| self, fields: Optional[List[str]] = None | |
| ) -> Union[dask.dataframe.DataFrame, pandas.DataFrame]: | |
| return self._partitions | |
| def read_partition( | |
| self, | |
| partition: int, | |
| fields: Optional[str] = None, | |
| ) -> Union[dask.dataframe.DataFrame, pandas.DataFrame]: | |
| return self._partitions[partition] | |
| def specs(self) -> List[Spec]: | |
| return self._specs | |
| def metadata(self) -> JSON: | |
| return self._metadata | |
| def __getitem__(self, key: str) -> ArrayAdapter: | |
| array = numpy.random.rand(2, 512, 512) | |
| metadata: JSON = {"foo": "bar"} | |
| structure = ArrayStructure( | |
| data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("int32")), | |
| shape=(2, 512, 512), | |
| chunks=((1, 1), (512,), (512,)), | |
| dims=("time", "x", "y"), # optional | |
| ) | |
| return CustomArrayAdapter(array, structure, metadata=metadata) | |
| def tableadapter_protocol_functions( | |
| adapter: TableAdapter, partition: int, fields: List[str] | |
| ) -> None: | |
| adapter.structure() | |
| adapter.read(fields) | |
| adapter.read_partition(partition) | |
| adapter.specs() | |
| adapter.metadata() | |
| adapter["abc"] | |
| def test_tableadapter_protocol(mocker: MockFixture) -> None: | |
| mock_call = mocker.patch.object(CustomTableAdapter, "structure") | |
| mock_call2 = mocker.patch.object(CustomTableAdapter, "read") | |
| mock_call3 = mocker.patch.object(CustomTableAdapter, "read_partition") | |
| mock_call4 = mocker.patch.object(CustomTableAdapter, "specs") | |
| mock_call5 = mocker.patch.object(CustomTableAdapter, "metadata") | |
| mock_call6 = mocker.patch.object(CustomTableAdapter, "__getitem__") | |
| structure = TableStructure( | |
| arrow_schema="a", npartitions=1, columns=["A"], resizable=False | |
| ) | |
| partitions = pandas.DataFrame([1]) | |
| metadata: JSON = {"foo": "bar"} | |
| fields = ["a", "b", "c"] | |
| partition = 1 | |
| anytableadapter = CustomTableAdapter(partitions, structure, metadata=metadata) | |
| assert anytableadapter.structure_family == StructureFamily.table | |
| tableadapter_protocol_functions(anytableadapter, partition, fields) | |
| mock_call.assert_called_once() | |
| mock_call2.assert_called_once_with(fields) | |
| mock_call3.assert_called_once_with(partition) | |
| mock_call4.assert_called_once() | |
| mock_call5.assert_called_once() | |
| mock_call6.assert_called_once_with("abc") | |
| class CustomAccessPolicy(AccessPolicy): | |
| def __init__(self, scopes: Optional[Scopes] = None) -> None: | |
| self.scopes = scopes if (scopes is not None) else ALL_SCOPES | |
| def _get_id(self, principal: Principal) -> None: | |
| return None | |
| async def init_node( | |
| self, | |
| principal: Principal, | |
| authn_access_tags: Optional[AccessTags], | |
| authn_scopes: Scopes, | |
| access_blob: Optional[AccessBlob] = None, | |
| ) -> Tuple[bool, Optional[AccessBlob]]: | |
| return (False, access_blob) | |
| async def modify_node( | |
| self, | |
| node: BaseAdapter, | |
| principal: Principal, | |
| authn_access_tags: Optional[AccessTags], | |
| authn_scopes: Scopes, | |
| access_blob: Optional[AccessBlob] = None, | |
| ) -> Tuple[bool, Optional[AccessBlob]]: | |
| return (False, access_blob) | |
| async def allowed_scopes( | |
| self, | |
| node: BaseAdapter, | |
| principal: Principal, | |
| authn_access_tags: Optional[AccessTags], | |
| authn_scopes: Scopes, | |
| ) -> Scopes: | |
| allowed = self.scopes | |
| somemetadata = node.metadata() # noqa: F841 | |
| return allowed | |
| async def filters( | |
| self, | |
| node: BaseAdapter, | |
| principal: Principal, | |
| authn_access_tags: Optional[AccessTags], | |
| authn_scopes: Scopes, | |
| scopes: Scopes, | |
| ) -> Filters: | |
| queries: Filters = ALL_ACCESS | |
| somespecs = node.specs() # noqa: F841 | |
| return queries | |
| async def accesspolicy_protocol_functions( | |
| policy: AccessPolicy, | |
| node: BaseAdapter, | |
| principal: Principal, | |
| authn_access_tags: Optional[AccessTags], | |
| authn_scopes: Scopes, | |
| scopes: Scopes, | |
| ) -> None: | |
| await policy.allowed_scopes(node, principal, authn_access_tags, authn_scopes) | |
| await policy.filters(node, principal, authn_access_tags, authn_scopes, scopes) | |
| # type: ignore | |
| async def test_accesspolicy_protocol(mocker: MockFixture) -> None: | |
| mock_call = mocker.patch.object(CustomAwkwardAdapter, "metadata") | |
| mock_call2 = mocker.patch.object(CustomAwkwardAdapter, "specs") | |
| anyaccesspolicy = CustomAccessPolicy(scopes={"a12mdjnk4"}) | |
| structure = AwkwardStructure(length=2, form={"a": "b"}) | |
| metadata: JSON = {"foo": "bar"} | |
| container = {"a": b"data"} | |
| principal = Principal( | |
| uuid="12345678124123412345678123456781", type=PrincipalType.user | |
| ) | |
| authn_access_tags = {"qux", "quux"} | |
| authn_scopes = {"abc", "baz"} | |
| scopes = {"abc"} | |
| anyawkwardadapter = CustomAwkwardAdapter(container, structure, metadata=metadata) | |
| await accesspolicy_protocol_functions( | |
| anyaccesspolicy, | |
| anyawkwardadapter, | |
| principal, | |
| authn_access_tags, | |
| authn_scopes, | |
| scopes, | |
| ) | |
| mock_call.assert_called_once() | |
| mock_call2.assert_called_once() | |
Xet Storage Details
- Size:
- 17.9 kB
- Xet hash:
- b090251a2aa779dd77cceecdc465a5dacf78ffa7fa9f35226ec226e9fe3934f5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.