Buckets:

download
raw
18 kB
import math
import string
import warnings
from pathlib import Path
import dask.array
import httpx
import numpy
import pytest
from starlette.status import (
HTTP_406_NOT_ACCEPTABLE,
HTTP_422_UNPROCESSABLE_CONTENT,
HTTP_503_SERVICE_UNAVAILABLE,
)
from tiled.adapters.array import ArrayAdapter
from tiled.adapters.mapping import MapAdapter
from tiled.adapters.utils import (
DataNotReadyError,
IncompatibleShapeError,
force_reshape,
)
from tiled.client import Context, from_context, record_history
from tiled.client.array import ArrayClient
from tiled.ndslice import NDSlice
from tiled.serialization.array import as_buffer
from tiled.server.app import build_app
from tiled.structures.array import ArrayStructure
from .utils import fail_with_status_code
array_cases = {
"b": (numpy.arange(10) % 2).astype("b"),
"i": numpy.arange(-10, 10, dtype="i"),
"uint8": numpy.arange(10, dtype="uint8"),
"uint16": numpy.arange(10, dtype="uint16"),
"uint64": numpy.arange(10, dtype="uint64"),
"f": numpy.arange(10, dtype="f"),
"c": (numpy.arange(10) * 1j).astype("c"),
"m": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]")
- numpy.datetime64("2008-01-01"),
"M": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]"),
"S": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="S3"),
"U": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="U3"),
}
# TODO bitfield "t", void "v", and object "O" (which is not supported by default)
scalar_cases = {k: numpy.array(v[0], dtype=v.dtype) for k, v in array_cases.items()}
for v in scalar_cases.values():
assert v.shape == ()
array_tree = MapAdapter({k: ArrayAdapter.from_array(v) for k, v in array_cases.items()})
scalar_tree = MapAdapter(
{k: ArrayAdapter.from_array(v) for k, v in scalar_cases.items()}
)
cube_cases = {
"tiny_cube": numpy.random.random((10, 10, 10)),
"tiny_hypercube": numpy.random.random((10, 10, 10, 10, 10)),
"chunked": dask.array.from_array(
numpy.arange(1_200_000, dtype="uint64").reshape((10, 300, 400)),
chunks=(1, 300, 200),
),
}
cube_tree = MapAdapter({k: ArrayAdapter.from_array(v) for k, v in cube_cases.items()})
inf_tree = MapAdapter(
{
"example": ArrayAdapter.from_array(
numpy.array([0, 1, numpy.nan, -numpy.inf, numpy.inf]),
metadata={"infinity": math.inf, "-infinity": -math.inf, "nan": numpy.nan},
)
},
metadata={"infinity": math.inf, "-infinity": -math.inf, "nan": numpy.nan},
)
arr_with_zero_dim = numpy.array([]).reshape((0, 100, 1, 10))
# Suppress RuntimeWarning: divide by zero encountered in true_divide
# from dask.array.core.
with warnings.catch_warnings():
zero_tree = MapAdapter(
{
"example": ArrayAdapter.from_array(
dask.array.from_array(arr_with_zero_dim, chunks=arr_with_zero_dim.shape)
)
}
)
nd_array = numpy.arange(9).reshape((3, 3))
uniform_array = numpy.empty((3,), dtype=object)
for i in range(uniform_array.shape[0]):
uniform_array[i] = nd_array[i]
irregular_array = numpy.array([numpy.arange(3), numpy.arange(4, 10)], dtype=object)
object_array = numpy.full((10,), {"a": 1}, dtype=object)
nested_arrays_tree = MapAdapter(
{
"uniform": ArrayAdapter.from_array(uniform_array),
"irregular": ArrayAdapter.from_array(irregular_array),
"objects": ArrayAdapter.from_array(object_array),
}
)
@pytest.fixture(scope="module")
def context():
tree = MapAdapter(
{
"array": array_tree,
"cube": cube_tree,
"inf": inf_tree,
"scalar": scalar_tree,
"zero": zero_tree,
"nested_arrays": nested_arrays_tree,
}
)
app = build_app(tree)
with Context.from_app(app) as context:
yield context
@pytest.mark.parametrize("kind", list(array_cases))
def test_array_dtypes(kind, context):
client = from_context(context)["array"]
expected = array_cases[kind]
actual_via_slice = client[kind][:]
actual_via_read = client[kind].read()
assert numpy.array_equal(actual_via_slice, actual_via_read)
assert numpy.array_equal(actual_via_slice, expected)
# Exercise a range of 1-D slicing patterns end-to-end for every dtype:
# contiguous, negative-index, negative-start, strided, full reversal, and
# scalar (integer) indexing.
for index in (
slice(2, 8),
slice(-3, None),
slice(None, None, 2),
slice(None, None, -1),
slice(8, 1, -1),
0,
-1,
):
assert numpy.array_equal(client[kind][index], expected[index]), index
@pytest.mark.parametrize("kind", list(scalar_cases))
def test_scalar_dtypes(kind, context):
client = from_context(context)["scalar"]
expected = scalar_cases[kind]
actual = client[kind].read()
assert numpy.array_equal(actual, expected)
def test_shape_with_zero(context):
client = from_context(context)["zero"]
actual = client["example"].read()
assert numpy.array_equal(actual, arr_with_zero_dim)
def test_nan_infinity_handler(tmpdir, context):
client = from_context(context)["inf"]
data = client["example"].read()
assert numpy.isnan(data).any()
assert numpy.isinf(data).any()
metadata = tuple(client["example"].metadata.values())
assert numpy.isnan(metadata).any()
assert numpy.isinf(metadata).any()
Path(tmpdir, "testjson").mkdir()
client["example"].export(Path(tmpdir, "testjson", "test.json"))
import json
def strict_parse_constant(c):
raise ValueError(f"{c} is not valid JSON")
with open(Path(tmpdir, "testjson", "test.json"), "r") as json_file:
open_json = json.load(json_file, parse_constant=strict_parse_constant)
expected_list = [0.0, 1.0, None, None, None]
assert open_json == expected_list
def test_block_validation(context):
"Verify that block is correctly specified."
client = from_context(context, "dask")["cube"]["tiny_cube"]
block_url = httpx.URL(client.item["links"]["block"])
# Malformed because it has only 2 dimensions, not 3.
malformed_block_url = block_url.copy_with(params={"block": "0,0"})
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
client.context.http_client.get(malformed_block_url).raise_for_status()
# Malformed because it has 4 dimensions, not 3.
malformed_block_url = block_url.copy_with(params={"block": "0,0,0,0"})
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
client.context.http_client.get(malformed_block_url).raise_for_status()
def test_dask(context):
expected = cube_cases["tiny_cube"]
client = from_context(context, "dask")["cube"]["tiny_cube"]
assert numpy.array_equal(client.read().compute(), expected)
assert numpy.array_equal(client.compute(), expected)
assert numpy.array_equal(client[:].compute(), expected)
def test_array_format_shape_from_cube(context):
client = from_context(context)["cube"]
with fail_with_status_code(HTTP_406_NOT_ACCEPTABLE):
client["tiny_hypercube"].export("test.png")
@pytest.mark.parametrize(
"bytesize_limit, num_gets_expected",
[
(None, 1), # Default, Entire array fits in one response
(300 * 400 * 8, 10), # Each frame fits in one response
(300 * 400 * 8 - 1, 20), # Just under the limit, each frame is split in half
(300 * 400 * 16, 5), # Two frames fit in one response
(300 * 100 * 8, 40), # Each chunk is split in half
],
)
def test_request_chunking(context, bytesize_limit, num_gets_expected, monkeypatch):
# Try reading a (10, 300, 400) array with (1, 300, 200) chunks and count requests
bytesize_limit = bytesize_limit or ArrayClient.RESPONSE_BYTESIZE_LIMIT
monkeypatch.setattr(ArrayClient, "RESPONSE_BYTESIZE_LIMIT", bytesize_limit)
client = from_context(context)["cube/chunked"]
with record_history() as h:
arr = client.read()
num_gets = sum(1 for entry in h.requests if entry.method == "GET")
assert num_gets == num_gets_expected
assert all("/array/full" in req.url.path for req in h.requests)
numpy.testing.assert_equal(arr, cube_cases["chunked"])
@pytest.mark.parametrize(
"index",
[
# (multi-dim) one slice that requires data from all chunks
(slice(None), 42, slice(100, 300)),
# integer indexing, positive and negative
5,
-1,
# negative start / stop
slice(-3, None),
slice(2, -2),
# strided, positive and negative step (incl. full reversal reaching 0)
slice(None, None, 2),
slice(None, None, -1),
slice(8, 1, -1),
# mixed multi-dim: strided + integer + reversed
(slice(None, None, 2), 42, slice(300, 100, -1)),
(slice(2, 8, 3), slice(None, None, -2), 100),
# Ellipsis combined with integer indexing
(Ellipsis, 0),
(0, Ellipsis),
# empty-result slices (server is not contacted; short-circuited)
slice(5, 5),
slice(10, 3),
],
)
def test_request_slicing(context, index):
# Each of these fits within a single response, so the client should make at
# most one GET (and none at all when the result is empty).
client = from_context(context)["cube/chunked"]
expected = numpy.asarray(cube_cases["chunked"])[index]
with record_history() as h:
actual = numpy.asarray(client[index])
numpy.testing.assert_equal(actual, expected)
num_gets = sum(1 for entry in h.requests if entry.method == "GET")
assert num_gets == (0 if expected.size == 0 else 1)
@pytest.mark.parametrize(
"index",
[
# positive-step slices split cleanly across chunk boundaries
(slice(None), slice(None), slice(100, 300)),
slice(None, None, 2),
slice(1, 9, 3),
slice(-3, None),
# reversed but stopping above 0 -> positive canonical stop
slice(8, 1, -1),
(slice(None, None, -2), slice(None, None, 2)),
# reversed reaching 0 -> negative canonical stop
slice(None, None, -1),
slice(5, None, -1),
(slice(None), slice(None, None, -1), slice(None)),
],
)
def test_request_slicing_chunked_response(context, index, monkeypatch):
# Force the multi-request split path by shrinking the per-response limit so
# that each frame (300 x 400) is fetched separately, then verify the
# reassembled result matches NumPy for a variety of slices.
monkeypatch.setattr(ArrayClient, "RESPONSE_BYTESIZE_LIMIT", 300 * 400 * 8)
client = from_context(context)["cube/chunked"]
expected = numpy.asarray(cube_cases["chunked"])[index]
with record_history() as h:
actual = numpy.asarray(client[index])
numpy.testing.assert_equal(actual, expected)
# This slice spans multiple chunks, so more than one GET is expected.
num_gets = sum(1 for entry in h.requests if entry.method == "GET")
assert num_gets > 1
@pytest.mark.parametrize("index", [10, -11, (0, 0, 400)])
def test_out_of_range_index_raises(context, index):
# Out-of-range integer indexing should raise IndexError, matching NumPy.
client = from_context(context)["cube/chunked"]
with pytest.raises(IndexError):
client[index]
def test_request_empty_slice(context):
# When reading an entire array, `slice=` should not be requested
client = from_context(context)["cube/chunked"]
with record_history() as h:
client.read()
client.read(slice=None)
client.read(slice=())
client.read(slice=Ellipsis)
client.read(slice=NDSlice())
client[...]
client[()]
client[:, :, :]
client[:]
client[:, ..., :]
assert len(h.requests) == 10
assert all("expected_shape" in req.url.params for req in h.requests)
assert all("slice" not in req.url.params for req in h.requests)
def test_array_interface(context):
client = from_context(context)["array"]
for k, v in client.items():
assert v.shape == array_cases[k].shape
assert v.ndim == array_cases[k].ndim
assert v.nbytes == array_cases[k].nbytes
assert v.dtype == array_cases[k].dtype
assert numpy.array_equal(numpy.asarray(v), array_cases[k])
# smoke test
v.chunks
v.dims
def test_uniform_nested_array_projected_to_ndarray(context):
client = from_context(context)["nested_arrays"]["uniform"]
assert client.dtype == numpy.int_
assert client.read().dtype == numpy.int_
assert numpy.array_equal(client.read(), nd_array)
@pytest.mark.parametrize("kind", ["irregular", "objects"])
def test_unparsable_nested_array_stringified(kind, context):
# This behavior is due to the fact that irregular Numpy arrays, and those with
# non-numeric types (except for strings) will likely have dtype=object,
# which may not be parsable or reducible. As such we fallback to taking the
# string representations of the array elements.
client = from_context(context)["nested_arrays"][kind]
assert "<U" in client.dtype.str
assert "<U" in client.read().dtype.str
assert client[0].dtype.kind == "U"
assert isinstance(client[0].item(), str)
@pytest.mark.parametrize("kind", list(array_cases))
def test_as_buffer(kind):
output = as_buffer("application/octet-stream", array_cases[kind], {})
assert len(output) == len(bytes(output))
@pytest.mark.parametrize(
"chunks, expected",
[
((3, 13, 17), "((3, 3, 3, 3, 3), (13,), (17,))"),
((1, 13, 17), "((1, 1, ..., 1), (13,), (17,))"),
((2, 13, 17), "((2, 2, ..., 2, 1), (13,), (17,))"),
((15, 13, 17), "((15,), (13,), (17,))"),
(((1, 1, 1, 1, 1, 2, 2, 2, 2, 2), 13, 17), "(variable, (13,), (17,))"),
],
)
def test_array_client_repr(tmpdir, chunks, expected):
arr = dask.array.random.random(size=(15, 13, 17), chunks=chunks)
adapter = MapAdapter({"arr": ArrayAdapter.from_array(arr, dims=("x", "y", "z"))})
app = build_app(adapter)
with Context.from_app(app) as context:
client = from_context(context)
rep = repr(client["arr"])
assert rep.startswith("<ArrayClient")
assert "shape=(15, 13, 17)" in rep
assert f"dtype={client['arr'].dtype}" in rep
assert f"chunks={expected}" in rep
if client["arr"].dims:
assert "dims=('x', 'y', 'z')" in rep
def test_force_reshape_noop_and_genuine_reshape():
# Exact match is a no-op.
arr = numpy.arange(6).reshape((2, 3))
assert force_reshape(arr, (2, 3)) is arr
# Same total size is a genuine, well-defined reshape.
reshaped = force_reshape(arr, (3, 2))
assert reshaped.shape == (3, 2)
numpy.testing.assert_array_equal(reshaped.ravel(), arr.ravel())
def test_force_reshape_trims_leading_axis_when_file_ahead():
# The file has grown along the leading axis beyond the advertised shape,
# with trailing dims unchanged. Serve exactly the advertised shape by
# trimming the extra leading frames.
arr = numpy.arange(4 * 3).reshape((4, 3))
trimmed = force_reshape(arr, (2, 3))
assert trimmed.shape == (2, 3)
numpy.testing.assert_array_equal(trimmed, arr[:2])
def test_force_reshape_raises_when_structure_ahead_of_data():
# The structure advertises more data than is available (catalog ahead of
# file, e.g. during a streaming append). We must not fabricate data. This
# is transient: more frames may still arrive.
arr = numpy.ones((3, 2, 4))
with pytest.raises(DataNotReadyError):
force_reshape(arr, (5, 2, 4))
def test_force_reshape_raises_on_incompatible_shape():
# Trailing dims differ and total size differs: a genuine incompatibility
# that retrying cannot resolve.
arr = numpy.ones((4, 3))
with pytest.raises(IncompatibleShapeError):
force_reshape(arr, (4, 5))
def test_structure_ahead_of_data_returns_503():
# Serve an array whose stored structure claims more frames than the
# underlying data has, mimicking a catalog that is ahead of the file
# during a streaming append. The read should surface as a retryable
# HTTP 503, not an opaque 500.
data = numpy.ones((3, 2, 4))
ahead_structure = ArrayStructure.from_array(numpy.ones((5, 2, 4)))
adapter = ArrayAdapter(data, ahead_structure)
app = build_app(MapAdapter({"streaming": adapter}))
with Context.from_app(app) as ahead_context:
client = from_context(ahead_context)
with fail_with_status_code(HTTP_503_SERVICE_UNAVAILABLE):
client["streaming"].read()
def test_incompatible_structure_returns_422():
# Serve an array whose stored structure is incompatible with the data in a
# way that retrying cannot fix (trailing dimensions disagree). This is a
# permanent error, surfaced as HTTP 422.
data = numpy.ones((4, 3))
bad_structure = ArrayStructure.from_array(numpy.ones((4, 5)))
adapter = ArrayAdapter(data, bad_structure)
app = build_app(MapAdapter({"streaming": adapter}))
with Context.from_app(app) as bad_context:
client = from_context(bad_context)
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
client["streaming"].read()
def test_file_ahead_of_structure_serves_trimmed():
# Serve an array whose underlying data has more frames than the stored
# structure advertises. The client sees exactly the advertised shape.
data = numpy.arange(4 * 2 * 4).reshape((4, 2, 4))
behind_structure = ArrayStructure.from_array(numpy.ones((2, 2, 4)))
adapter = ArrayAdapter(data, behind_structure)
app = build_app(MapAdapter({"streaming": adapter}))
with Context.from_app(app) as behind_context:
client = from_context(behind_context)
result = client["streaming"].read()
assert result.shape == (2, 2, 4)
numpy.testing.assert_array_equal(result, data[:2])

Xet Storage Details

Size:
18 kB
·
Xet hash:
afc239a286a11d4343e56f9bd8025e0dd935172ac4a0b46f91a68f82d916faf4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.