AnonymousSpottedToad/securityPatternsRecovery / codebase /repos /python /bluesky_tiled /tests /test_array.py
| import math | |
| import string | |
| import warnings | |
| from pathlib import Path | |
| import dask.array | |
| import httpx | |
| import numpy | |
| import pytest | |
| from starlette.status import ( | |
| HTTP_406_NOT_ACCEPTABLE, | |
| HTTP_422_UNPROCESSABLE_CONTENT, | |
| HTTP_503_SERVICE_UNAVAILABLE, | |
| ) | |
| from tiled.adapters.array import ArrayAdapter | |
| from tiled.adapters.mapping import MapAdapter | |
| from tiled.adapters.utils import ( | |
| DataNotReadyError, | |
| IncompatibleShapeError, | |
| force_reshape, | |
| ) | |
| from tiled.client import Context, from_context, record_history | |
| from tiled.client.array import ArrayClient | |
| from tiled.ndslice import NDSlice | |
| from tiled.serialization.array import as_buffer | |
| from tiled.server.app import build_app | |
| from tiled.structures.array import ArrayStructure | |
| from .utils import fail_with_status_code | |
| array_cases = { | |
| "b": (numpy.arange(10) % 2).astype("b"), | |
| "i": numpy.arange(-10, 10, dtype="i"), | |
| "uint8": numpy.arange(10, dtype="uint8"), | |
| "uint16": numpy.arange(10, dtype="uint16"), | |
| "uint64": numpy.arange(10, dtype="uint64"), | |
| "f": numpy.arange(10, dtype="f"), | |
| "c": (numpy.arange(10) * 1j).astype("c"), | |
| "m": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]") | |
| - numpy.datetime64("2008-01-01"), | |
| "M": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]"), | |
| "S": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="S3"), | |
| "U": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="U3"), | |
| } | |
| # TODO bitfield "t", void "v", and object "O" (which is not supported by default) | |
| scalar_cases = {k: numpy.array(v[0], dtype=v.dtype) for k, v in array_cases.items()} | |
| for v in scalar_cases.values(): | |
| assert v.shape == () | |
| array_tree = MapAdapter({k: ArrayAdapter.from_array(v) for k, v in array_cases.items()}) | |
| scalar_tree = MapAdapter( | |
| {k: ArrayAdapter.from_array(v) for k, v in scalar_cases.items()} | |
| ) | |
| cube_cases = { | |
| "tiny_cube": numpy.random.random((10, 10, 10)), | |
| "tiny_hypercube": numpy.random.random((10, 10, 10, 10, 10)), | |
| "chunked": dask.array.from_array( | |
| numpy.arange(1_200_000, dtype="uint64").reshape((10, 300, 400)), | |
| chunks=(1, 300, 200), | |
| ), | |
| } | |
| cube_tree = MapAdapter({k: ArrayAdapter.from_array(v) for k, v in cube_cases.items()}) | |
| inf_tree = MapAdapter( | |
| { | |
| "example": ArrayAdapter.from_array( | |
| numpy.array([0, 1, numpy.nan, -numpy.inf, numpy.inf]), | |
| metadata={"infinity": math.inf, "-infinity": -math.inf, "nan": numpy.nan}, | |
| ) | |
| }, | |
| metadata={"infinity": math.inf, "-infinity": -math.inf, "nan": numpy.nan}, | |
| ) | |
| arr_with_zero_dim = numpy.array([]).reshape((0, 100, 1, 10)) | |
| # Suppress RuntimeWarning: divide by zero encountered in true_divide | |
| # from dask.array.core. | |
| with warnings.catch_warnings(): | |
| zero_tree = MapAdapter( | |
| { | |
| "example": ArrayAdapter.from_array( | |
| dask.array.from_array(arr_with_zero_dim, chunks=arr_with_zero_dim.shape) | |
| ) | |
| } | |
| ) | |
| nd_array = numpy.arange(9).reshape((3, 3)) | |
| uniform_array = numpy.empty((3,), dtype=object) | |
| for i in range(uniform_array.shape[0]): | |
| uniform_array[i] = nd_array[i] | |
| irregular_array = numpy.array([numpy.arange(3), numpy.arange(4, 10)], dtype=object) | |
| object_array = numpy.full((10,), {"a": 1}, dtype=object) | |
| nested_arrays_tree = MapAdapter( | |
| { | |
| "uniform": ArrayAdapter.from_array(uniform_array), | |
| "irregular": ArrayAdapter.from_array(irregular_array), | |
| "objects": ArrayAdapter.from_array(object_array), | |
| } | |
| ) | |
| def context(): | |
| tree = MapAdapter( | |
| { | |
| "array": array_tree, | |
| "cube": cube_tree, | |
| "inf": inf_tree, | |
| "scalar": scalar_tree, | |
| "zero": zero_tree, | |
| "nested_arrays": nested_arrays_tree, | |
| } | |
| ) | |
| app = build_app(tree) | |
| with Context.from_app(app) as context: | |
| yield context | |
| def test_array_dtypes(kind, context): | |
| client = from_context(context)["array"] | |
| expected = array_cases[kind] | |
| actual_via_slice = client[kind][:] | |
| actual_via_read = client[kind].read() | |
| assert numpy.array_equal(actual_via_slice, actual_via_read) | |
| assert numpy.array_equal(actual_via_slice, expected) | |
| # Exercise a range of 1-D slicing patterns end-to-end for every dtype: | |
| # contiguous, negative-index, negative-start, strided, full reversal, and | |
| # scalar (integer) indexing. | |
| for index in ( | |
| slice(2, 8), | |
| slice(-3, None), | |
| slice(None, None, 2), | |
| slice(None, None, -1), | |
| slice(8, 1, -1), | |
| 0, | |
| -1, | |
| ): | |
| assert numpy.array_equal(client[kind][index], expected[index]), index | |
| def test_scalar_dtypes(kind, context): | |
| client = from_context(context)["scalar"] | |
| expected = scalar_cases[kind] | |
| actual = client[kind].read() | |
| assert numpy.array_equal(actual, expected) | |
| def test_shape_with_zero(context): | |
| client = from_context(context)["zero"] | |
| actual = client["example"].read() | |
| assert numpy.array_equal(actual, arr_with_zero_dim) | |
| def test_nan_infinity_handler(tmpdir, context): | |
| client = from_context(context)["inf"] | |
| data = client["example"].read() | |
| assert numpy.isnan(data).any() | |
| assert numpy.isinf(data).any() | |
| metadata = tuple(client["example"].metadata.values()) | |
| assert numpy.isnan(metadata).any() | |
| assert numpy.isinf(metadata).any() | |
| Path(tmpdir, "testjson").mkdir() | |
| client["example"].export(Path(tmpdir, "testjson", "test.json")) | |
| import json | |
| def strict_parse_constant(c): | |
| raise ValueError(f"{c} is not valid JSON") | |
| with open(Path(tmpdir, "testjson", "test.json"), "r") as json_file: | |
| open_json = json.load(json_file, parse_constant=strict_parse_constant) | |
| expected_list = [0.0, 1.0, None, None, None] | |
| assert open_json == expected_list | |
| def test_block_validation(context): | |
| "Verify that block is correctly specified." | |
| client = from_context(context, "dask")["cube"]["tiny_cube"] | |
| block_url = httpx.URL(client.item["links"]["block"]) | |
| # Malformed because it has only 2 dimensions, not 3. | |
| malformed_block_url = block_url.copy_with(params={"block": "0,0"}) | |
| with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT): | |
| client.context.http_client.get(malformed_block_url).raise_for_status() | |
| # Malformed because it has 4 dimensions, not 3. | |
| malformed_block_url = block_url.copy_with(params={"block": "0,0,0,0"}) | |
| with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT): | |
| client.context.http_client.get(malformed_block_url).raise_for_status() | |
| def test_dask(context): | |
| expected = cube_cases["tiny_cube"] | |
| client = from_context(context, "dask")["cube"]["tiny_cube"] | |
| assert numpy.array_equal(client.read().compute(), expected) | |
| assert numpy.array_equal(client.compute(), expected) | |
| assert numpy.array_equal(client[:].compute(), expected) | |
| def test_array_format_shape_from_cube(context): | |
| client = from_context(context)["cube"] | |
| with fail_with_status_code(HTTP_406_NOT_ACCEPTABLE): | |
| client["tiny_hypercube"].export("test.png") | |
| def test_request_chunking(context, bytesize_limit, num_gets_expected, monkeypatch): | |
| # Try reading a (10, 300, 400) array with (1, 300, 200) chunks and count requests | |
| bytesize_limit = bytesize_limit or ArrayClient.RESPONSE_BYTESIZE_LIMIT | |
| monkeypatch.setattr(ArrayClient, "RESPONSE_BYTESIZE_LIMIT", bytesize_limit) | |
| client = from_context(context)["cube/chunked"] | |
| with record_history() as h: | |
| arr = client.read() | |
| num_gets = sum(1 for entry in h.requests if entry.method == "GET") | |
| assert num_gets == num_gets_expected | |
| assert all("/array/full" in req.url.path for req in h.requests) | |
| numpy.testing.assert_equal(arr, cube_cases["chunked"]) | |
| def test_request_slicing(context, index): | |
| # Each of these fits within a single response, so the client should make at | |
| # most one GET (and none at all when the result is empty). | |
| client = from_context(context)["cube/chunked"] | |
| expected = numpy.asarray(cube_cases["chunked"])[index] | |
| with record_history() as h: | |
| actual = numpy.asarray(client[index]) | |
| numpy.testing.assert_equal(actual, expected) | |
| num_gets = sum(1 for entry in h.requests if entry.method == "GET") | |
| assert num_gets == (0 if expected.size == 0 else 1) | |
| def test_request_slicing_chunked_response(context, index, monkeypatch): | |
| # Force the multi-request split path by shrinking the per-response limit so | |
| # that each frame (300 x 400) is fetched separately, then verify the | |
| # reassembled result matches NumPy for a variety of slices. | |
| monkeypatch.setattr(ArrayClient, "RESPONSE_BYTESIZE_LIMIT", 300 * 400 * 8) | |
| client = from_context(context)["cube/chunked"] | |
| expected = numpy.asarray(cube_cases["chunked"])[index] | |
| with record_history() as h: | |
| actual = numpy.asarray(client[index]) | |
| numpy.testing.assert_equal(actual, expected) | |
| # This slice spans multiple chunks, so more than one GET is expected. | |
| num_gets = sum(1 for entry in h.requests if entry.method == "GET") | |
| assert num_gets > 1 | |
| def test_out_of_range_index_raises(context, index): | |
| # Out-of-range integer indexing should raise IndexError, matching NumPy. | |
| client = from_context(context)["cube/chunked"] | |
| with pytest.raises(IndexError): | |
| client[index] | |
| def test_request_empty_slice(context): | |
| # When reading an entire array, `slice=` should not be requested | |
| client = from_context(context)["cube/chunked"] | |
| with record_history() as h: | |
| client.read() | |
| client.read(slice=None) | |
| client.read(slice=()) | |
| client.read(slice=Ellipsis) | |
| client.read(slice=NDSlice()) | |
| client[...] | |
| client[()] | |
| client[:, :, :] | |
| client[:] | |
| client[:, ..., :] | |
| assert len(h.requests) == 10 | |
| assert all("expected_shape" in req.url.params for req in h.requests) | |
| assert all("slice" not in req.url.params for req in h.requests) | |
| def test_array_interface(context): | |
| client = from_context(context)["array"] | |
| for k, v in client.items(): | |
| assert v.shape == array_cases[k].shape | |
| assert v.ndim == array_cases[k].ndim | |
| assert v.nbytes == array_cases[k].nbytes | |
| assert v.dtype == array_cases[k].dtype | |
| assert numpy.array_equal(numpy.asarray(v), array_cases[k]) | |
| # smoke test | |
| v.chunks | |
| v.dims | |
| def test_uniform_nested_array_projected_to_ndarray(context): | |
| client = from_context(context)["nested_arrays"]["uniform"] | |
| assert client.dtype == numpy.int_ | |
| assert client.read().dtype == numpy.int_ | |
| assert numpy.array_equal(client.read(), nd_array) | |
| def test_unparsable_nested_array_stringified(kind, context): | |
| # This behavior is due to the fact that irregular Numpy arrays, and those with | |
| # non-numeric types (except for strings) will likely have dtype=object, | |
| # which may not be parsable or reducible. As such we fallback to taking the | |
| # string representations of the array elements. | |
| client = from_context(context)["nested_arrays"][kind] | |
| assert "<U" in client.dtype.str | |
| assert "<U" in client.read().dtype.str | |
| assert client[0].dtype.kind == "U" | |
| assert isinstance(client[0].item(), str) | |
| def test_as_buffer(kind): | |
| output = as_buffer("application/octet-stream", array_cases[kind], {}) | |
| assert len(output) == len(bytes(output)) | |
| def test_array_client_repr(tmpdir, chunks, expected): | |
| arr = dask.array.random.random(size=(15, 13, 17), chunks=chunks) | |
| adapter = MapAdapter({"arr": ArrayAdapter.from_array(arr, dims=("x", "y", "z"))}) | |
| app = build_app(adapter) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| rep = repr(client["arr"]) | |
| assert rep.startswith("<ArrayClient") | |
| assert "shape=(15, 13, 17)" in rep | |
| assert f"dtype={client['arr'].dtype}" in rep | |
| assert f"chunks={expected}" in rep | |
| if client["arr"].dims: | |
| assert "dims=('x', 'y', 'z')" in rep | |
| def test_force_reshape_noop_and_genuine_reshape(): | |
| # Exact match is a no-op. | |
| arr = numpy.arange(6).reshape((2, 3)) | |
| assert force_reshape(arr, (2, 3)) is arr | |
| # Same total size is a genuine, well-defined reshape. | |
| reshaped = force_reshape(arr, (3, 2)) | |
| assert reshaped.shape == (3, 2) | |
| numpy.testing.assert_array_equal(reshaped.ravel(), arr.ravel()) | |
| def test_force_reshape_trims_leading_axis_when_file_ahead(): | |
| # The file has grown along the leading axis beyond the advertised shape, | |
| # with trailing dims unchanged. Serve exactly the advertised shape by | |
| # trimming the extra leading frames. | |
| arr = numpy.arange(4 * 3).reshape((4, 3)) | |
| trimmed = force_reshape(arr, (2, 3)) | |
| assert trimmed.shape == (2, 3) | |
| numpy.testing.assert_array_equal(trimmed, arr[:2]) | |
| def test_force_reshape_raises_when_structure_ahead_of_data(): | |
| # The structure advertises more data than is available (catalog ahead of | |
| # file, e.g. during a streaming append). We must not fabricate data. This | |
| # is transient: more frames may still arrive. | |
| arr = numpy.ones((3, 2, 4)) | |
| with pytest.raises(DataNotReadyError): | |
| force_reshape(arr, (5, 2, 4)) | |
| def test_force_reshape_raises_on_incompatible_shape(): | |
| # Trailing dims differ and total size differs: a genuine incompatibility | |
| # that retrying cannot resolve. | |
| arr = numpy.ones((4, 3)) | |
| with pytest.raises(IncompatibleShapeError): | |
| force_reshape(arr, (4, 5)) | |
| def test_structure_ahead_of_data_returns_503(): | |
| # Serve an array whose stored structure claims more frames than the | |
| # underlying data has, mimicking a catalog that is ahead of the file | |
| # during a streaming append. The read should surface as a retryable | |
| # HTTP 503, not an opaque 500. | |
| data = numpy.ones((3, 2, 4)) | |
| ahead_structure = ArrayStructure.from_array(numpy.ones((5, 2, 4))) | |
| adapter = ArrayAdapter(data, ahead_structure) | |
| app = build_app(MapAdapter({"streaming": adapter})) | |
| with Context.from_app(app) as ahead_context: | |
| client = from_context(ahead_context) | |
| with fail_with_status_code(HTTP_503_SERVICE_UNAVAILABLE): | |
| client["streaming"].read() | |
| def test_incompatible_structure_returns_422(): | |
| # Serve an array whose stored structure is incompatible with the data in a | |
| # way that retrying cannot fix (trailing dimensions disagree). This is a | |
| # permanent error, surfaced as HTTP 422. | |
| data = numpy.ones((4, 3)) | |
| bad_structure = ArrayStructure.from_array(numpy.ones((4, 5))) | |
| adapter = ArrayAdapter(data, bad_structure) | |
| app = build_app(MapAdapter({"streaming": adapter})) | |
| with Context.from_app(app) as bad_context: | |
| client = from_context(bad_context) | |
| with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT): | |
| client["streaming"].read() | |
| def test_file_ahead_of_structure_serves_trimmed(): | |
| # Serve an array whose underlying data has more frames than the stored | |
| # structure advertises. The client sees exactly the advertised shape. | |
| data = numpy.arange(4 * 2 * 4).reshape((4, 2, 4)) | |
| behind_structure = ArrayStructure.from_array(numpy.ones((2, 2, 4))) | |
| adapter = ArrayAdapter(data, behind_structure) | |
| app = build_app(MapAdapter({"streaming": adapter})) | |
| with Context.from_app(app) as behind_context: | |
| client = from_context(behind_context) | |
| result = client["streaming"].read() | |
| assert result.shape == (2, 2, 4) | |
| numpy.testing.assert_array_equal(result, data[:2]) | |
Xet Storage Details
- Size:
- 18 kB
- Xet hash:
- afc239a286a11d4343e56f9bd8025e0dd935172ac4a0b46f91a68f82d916faf4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.