Buckets:

download
raw
16.2 kB
from pathlib import Path
import numpy
import pytest
import tifffile as tf
from tiled.adapters.mapping import MapAdapter
from tiled.adapters.tiff import TiffAdapter, TiffSequenceAdapter
from tiled.catalog import in_memory
from tiled.client import Context, from_context
from tiled.client.register import IMG_SEQUENCE_EMPTY_NAME_ROOT, register
from tiled.server.app import build_app
from tiled.structures.array import ArrayStructure, BuiltinDtype
from tiled.utils import ensure_uri
COLOR_SHAPE = (11, 17, 3)
rng = numpy.random.default_rng(12345)
tiff_data = rng.integers(0, 255, size=(12, 5, 7, 4), dtype="uint8")
color_data = rng.integers(0, 255, size=COLOR_SHAPE, dtype="uint8")
@pytest.fixture(scope="module")
def client(tmpdir_module):
sequence_directory = Path(tmpdir_module, "sequence")
sequence_directory.mkdir()
filepaths = []
for i in range(12):
filepath = sequence_directory / f"temp{i:05}.tif"
tf.imwrite(filepath, tiff_data[i, ...])
filepaths.append(filepath)
path = Path(tmpdir_module, "color.tif")
tf.imwrite(path, color_data)
tree = MapAdapter(
{
"color": TiffAdapter(ensure_uri(path)),
"sequence": TiffSequenceAdapter.from_uris(
*[ensure_uri(filepath) for filepath in filepaths]
),
# 5d sequence stacked along the leftmost dimension
"5d_sequence": TiffSequenceAdapter(
[ensure_uri(filepath) for filepath in filepaths[:3]],
structure=ArrayStructure(
shape=(3, 1, 5, 7, 4),
chunks=((1, 1, 1), (1,), (5,), (7,), (4,)),
data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")),
),
),
# 5d sequence stacked along the second dimension
"5d_sequence_second_dim": TiffSequenceAdapter(
[ensure_uri(filepath) for filepath in filepaths[:3]],
structure=ArrayStructure(
shape=(1, 3, 5, 7, 4),
chunks=((1,), (1,) * 3, (5,), (7,), (4,)),
data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")),
),
),
# 5d sequence stacked along the first and second dimensions
"5d_sequence_first_and_second_dim": TiffSequenceAdapter(
[ensure_uri(filepath) for filepath in filepaths],
structure=ArrayStructure(
shape=(3, 4, 5, 7, 4),
chunks=((1, 1, 1), (2, 2), (5,), (7,), (4,)),
data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")),
),
),
}
)
# `_chunks` would be provided in data_source.properties if initialized from a catalog
tree["5d_sequence"]._chunks = ((1, 1, 1), (5,), (7,), (4,))
tree["5d_sequence_second_dim"]._chunks = ((1, 1, 1), (5,), (7,), (4,))
tree["5d_sequence_first_and_second_dim"]._chunks = ((1,) * 12, (5,), (7,), (4,))
app = build_app(tree)
with Context.from_app(app) as context:
client = from_context(context)
yield client
@pytest.mark.parametrize(
"slice_input, correct_shape",
[
(None, (12, 5, 7, 4)),
(1, (5, 7, 4)),
(slice(0, 3, 2), (2, 5, 7, 4)),
((1, slice(0, 3), slice(0, 3)), (3, 3, 4)),
((slice(0, 3), slice(0, 3), slice(0, 3)), (3, 3, 3, 4)),
((..., 0, 0, 0), (12,)),
((0, slice(0, 1), slice(0, 2), ...), (1, 2, 4)),
((0, ..., slice(0, 2)), (5, 7, 2)),
((..., slice(0, 1)), (12, 5, 7, 1)),
],
)
def test_tiff_sequence(client, slice_input, correct_shape):
arr = client["sequence"].read(slice=slice_input)
assert arr.shape == correct_shape
true_arr = tiff_data[slice_input or ()]
numpy.testing.assert_equal(arr, true_arr)
@pytest.mark.parametrize(
"slice_input, correct_shape",
[
(None, (3, 1, 5, 7, 4)),
(..., (3, 1, 5, 7, 4)),
((), (3, 1, 5, 7, 4)),
(1, (1, 5, 7, 4)),
((slice(0, 10), 0), (3, 5, 7, 4)),
(slice(0, 3, 2), (2, 1, 5, 7, 4)),
((1, slice(0, 10), slice(0, 3), slice(0, 3)), (1, 3, 3, 4)),
((slice(0, 3), 0, slice(0, 3), slice(0, 3)), (3, 3, 3, 4)),
((..., 0, 0, 0, 0), (3,)),
((0, slice(0, 1), slice(0, 1), slice(0, 2), ...), (1, 1, 2, 4)),
((0, ..., slice(0, 2)), (1, 5, 7, 2)),
((..., slice(0, 1)), (3, 1, 5, 7, 1)),
],
)
def test_forced_reshaping(client, slice_input, correct_shape):
arr = client["5d_sequence"].read(slice=slice_input)
assert arr.shape == correct_shape
slice_input = () if slice_input is None else slice_input
true_arr = tiff_data[:3, ...].reshape(3, 1, 5, 7, 4)[slice_input]
numpy.testing.assert_equal(arr, true_arr)
@pytest.mark.parametrize(
"slice_input, correct_shape",
[
(None, (1, 3, 5, 7, 4)),
(..., (1, 3, 5, 7, 4)),
((), (1, 3, 5, 7, 4)),
(0, (3, 5, 7, 4)),
((slice(0, 10), 1), (1, 5, 7, 4)),
(slice(0, 3, 2), (1, 3, 5, 7, 4)),
((0, slice(0, 10), slice(0, 3), slice(0, 3)), (3, 3, 3, 4)),
((slice(0, 3), 0, slice(0, 3), slice(0, 3)), (1, 3, 3, 4)),
((..., 0, 0, 0, 0), (1,)),
((0, slice(0, 1), slice(0, 1), slice(0, 2), ...), (1, 1, 2, 4)),
((0, ..., slice(0, 2)), (3, 5, 7, 2)),
((..., slice(0, 1)), (1, 3, 5, 7, 1)),
],
)
def test_forced_reshaping_along_second_dimension(client, slice_input, correct_shape):
"""Test that the presence of a frame per point dimension does not affect
the ability to read with a slice that would normally cause reshaping.
"""
arr = client["5d_sequence_second_dim"].read(slice=slice_input)
assert arr.shape == correct_shape
slice_input = () if slice_input is None else slice_input
true_arr = tiff_data[:3, ...].reshape(1, 3, 5, 7, 4)[slice_input]
numpy.testing.assert_equal(arr, true_arr)
@pytest.mark.parametrize(
"slice_input, correct_shape",
[
(None, (3, 4, 5, 7, 4)),
(..., (3, 4, 5, 7, 4)),
((), (3, 4, 5, 7, 4)),
(0, (4, 5, 7, 4)),
((slice(0, 10), 1), (3, 5, 7, 4)),
(slice(0, 3, 2), (2, 4, 5, 7, 4)),
((0, slice(0, 10), slice(0, 3), slice(0, 3)), (4, 3, 3, 4)),
((slice(0, 3), 0, slice(0, 3), slice(0, 3)), (3, 3, 3, 4)),
((..., 0, 0, 0, 0), (3,)),
((0, slice(0, 1), slice(0, 1), slice(0, 2), ...), (1, 1, 2, 4)),
((0, ..., slice(0, 2)), (4, 5, 7, 2)),
((..., slice(0, 1)), (3, 4, 5, 7, 1)),
],
)
def test_forced_reshaping_along_first_and_second_dimensions(
client, slice_input, correct_shape
):
"""Test that the presence of a frame per point dimension does not affect
the ability to read with a slice that would normally cause reshaping.
"""
arr = client["5d_sequence_first_and_second_dim"].read(slice=slice_input)
assert arr.shape == correct_shape
slice_input = () if slice_input is None else slice_input
true_arr = tiff_data.reshape(3, 4, 5, 7, 4)[slice_input]
numpy.testing.assert_equal(arr, true_arr)
@pytest.mark.parametrize("block_input, correct_shape", [((0, 0, 0, 0), (1, 5, 7, 4))])
def test_tiff_sequence_block(client, block_input, correct_shape):
arr = client["sequence"].read_block(block_input)
assert arr.shape == correct_shape
@pytest.mark.parametrize(
"slice_input, correct_shape",
[
((slice(None), slice(None), 0, 0, 0), (3, 4)),
((..., 0, 0, 0, 0), (3,)),
((slice(None), slice(None), slice(None), 0, 0), (3, 4, 5)),
((slice(0, 2), slice(None), 1, 2, 3), (2, 4)),
((slice(None),) * 5, (3, 4, 5, 7, 4)),
],
)
def test_reshaped_sequence_reducing_slice_is_batch_invariant(
client, monkeypatch, slice_input, correct_shape
):
"""A slice that touches many files but keeps only part of each frame must be
read in memory-bounded batches without changing the result. Force a tiny
per-batch budget so every file lands in its own batch, and check the result
matches both numpy ground truth and a single-batch read.
"""
import tiled.adapters.sequence as sequence_module
true_arr = tiff_data.reshape(3, 4, 5, 7, 4)[slice_input]
# Single batch (generous budget).
monkeypatch.setattr(sequence_module, "READ_BATCH_BYTES", 1 << 30)
single = client["5d_sequence_first_and_second_dim"].read(slice=slice_input)
assert single.shape == correct_shape
numpy.testing.assert_equal(single, true_arr)
# Force one file per batch; result must be identical.
monkeypatch.setattr(sequence_module, "READ_BATCH_BYTES", 1)
batched = client["5d_sequence_first_and_second_dim"].read(slice=slice_input)
assert batched.shape == correct_shape
numpy.testing.assert_equal(batched, true_arr)
@pytest.mark.asyncio
async def test_tiff_sequence_order(tmpdir):
"""
directory/
00001.tif
00002.tif
...
00010.tif
"""
data = numpy.ones((4, 5))
num_files = 10
for i in range(num_files):
tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), data * i)
adapter = in_memory(readable_storage=[tmpdir])
with Context.from_app(build_app(adapter)) as context:
client = from_context(context)
await register(client, tmpdir)
for i in range(num_files):
numpy.testing.assert_equal(client["image"][i], data * i)
@pytest.mark.asyncio
async def test_tiff_sequence_with_directory_walker(tmpdir):
"""
directory/
00001.tif
00002.tif
...
00010.tif
single_image.tif
image00001.tif
image00002.tif
...
image00010.tif
other_image00001.tif
other_image00002.tif
...
other_image00010.tif
other_image2_00001.tif
other_image2_00002.tif
...
other_image2_00010.tif
other_file1.csv
other_file2.csv
stuff.csv
"""
data = numpy.random.random((3, 5))
for i in range(10):
tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), data)
tf.imwrite(Path(tmpdir / f"other_image{i:05}.tif"), data)
tf.imwrite(Path(tmpdir / f"{i:05}.tif"), data)
tf.imwrite(Path(tmpdir / f"other_image2_{i:05}.tif"), data)
tf.imwrite(Path(tmpdir / "single_image.tif"), data)
for target in ["stuff.csv", "other_file1.csv", "other_file2.csv"]:
with open(Path(tmpdir / target), "w") as file:
file.write(
"""
a,b,c
1,2,3
"""
)
adapter = in_memory(readable_storage=[tmpdir])
with Context.from_app(build_app(adapter)) as context:
client = from_context(context)
await register(client, tmpdir)
# Single image is its own node.
assert client["single_image"].shape == (3, 5)
# Each sequence is grouped into a node.
assert client[IMG_SEQUENCE_EMPTY_NAME_ROOT].shape == (10, 3, 5)
assert client["image"].shape == (10, 3, 5)
assert client["other_image"].shape == (10, 3, 5)
assert client["other_image2_"].shape == (10, 3, 5)
# The sequence grouping digit-only files appears with a uuid
named_keys = [
"single_image",
"image",
"other_image",
"other_image2_",
"other_file1",
"other_file2",
"stuff",
]
no_name_keys = [key for key in client.keys() if key not in named_keys]
# There is only a single one of this type
assert len(no_name_keys) == 1
assert client[no_name_keys[0]].shape == (10, 3, 5)
# Other files are single nodes.
assert client["stuff"].columns == ["a", "b", "c"]
assert client["other_file1"].columns == ["a", "b", "c"]
assert client["other_file2"].columns == ["a", "b", "c"]
@pytest.mark.asyncio
async def test_tiff_sequence_client_data_sources_assets(tmpdir):
"""A file-sequence node exposes ALL its assets to the client via
`c.data_sources()[0].assets`, even though the server loads assets
lazily (asset-free `data_sources` property + on-demand per-frame reads).
Also confirms single-frame reads return correct data, exercising the lazy
per-frame resolution path end-to-end through the catalog.
"""
num_frames = 10
base = numpy.random.default_rng(0).integers(0, 255, size=(3, 5), dtype="uint8")
frames = []
for i in range(num_frames):
arr = (base + i).astype("uint8")
tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), arr)
frames.append(arr)
adapter = in_memory(readable_storage=[tmpdir])
with Context.from_app(build_app(adapter)) as context:
client = from_context(context)
await register(client, tmpdir)
seq = client["image"]
assert seq.shape == (num_frames, 3, 5)
# Client-facing data_sources() must still expose every asset.
data_sources = seq.data_sources()
assert data_sources is not None
assert len(data_sources) == 1
assets = data_sources[0].assets
assert len(assets) == num_frames
assert all(asset.data_uri for asset in assets)
# Single-frame reads via the lazy path return the correct frame.
for i in (0, 3, num_frames - 1):
numpy.testing.assert_array_equal(seq[i], frames[i])
# A multi-frame slice also resolves correctly.
numpy.testing.assert_array_equal(seq[2:5], numpy.stack(frames[2:5]))
@pytest.mark.asyncio
async def test_tiff_sequence_lazy_http_block_and_slice_reads(tmpdir):
"""End-to-end HTTP exercise of the lazy per-frame path.
Reads reach the server via the `/array/full` (slice) and `/array/block`
routes, which dispatch to the catalog's lazy resolver. Confirms both give
data identical to the eager reference for whole-array, sliced, strided, and
single-block reads.
"""
num_frames = 6
base = numpy.random.default_rng(1).integers(0, 255, size=(3, 5), dtype="uint8")
frames = []
for i in range(num_frames):
arr = (base + i).astype("uint8")
tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), arr)
frames.append(arr)
stacked = numpy.stack(frames)
adapter = in_memory(readable_storage=[tmpdir])
with Context.from_app(build_app(adapter)) as context:
client = from_context(context)
await register(client, tmpdir)
seq = client["image"]
assert seq.shape == (num_frames, 3, 5)
# Full read over HTTP (/array/full).
numpy.testing.assert_array_equal(seq.read(), stacked)
# Contiguous and strided slice reads over HTTP.
numpy.testing.assert_array_equal(seq.read(slice=(slice(1, 4),)), stacked[1:4])
numpy.testing.assert_array_equal(
seq.read(slice=(slice(0, num_frames, 2),)), stacked[0:num_frames:2]
)
# Per-block reads over HTTP (/array/block): each file is one block on axis 0.
for i in (0, 2, num_frames - 1):
block = seq.read_block((i, 0, 0))
numpy.testing.assert_array_equal(block, frames[i][numpy.newaxis])
# Block read with a within-block slice (/array/block?slice=...): the block
# selects the file(s) to resolve; the within-block slice is applied to the
# already-loaded block data and must never change which files are read.
for i in (0, 3, num_frames - 1):
block = seq.read_block((i, 0, 0), slice=(0, slice(1, 3)))
numpy.testing.assert_array_equal(block, frames[i][1:3])
def test_rgb(client):
"Test an RGB TIFF."
arr = client["color"].read()
assert arr.shape == COLOR_SHAPE
def test_tiff_sequence_cache(client):
from numpy.testing import assert_raises
# The two requests go through the same method in the server (read_block) to
# call the same object
indexed_array = client["sequence"][0]
read_array = client["sequence"].read(0)
# Using a different index to confirm that the previous cache doesn't affect the new array
other_read_array = client["sequence"].read(1)
numpy.testing.assert_equal(indexed_array, read_array)
assert_raises(
AssertionError, numpy.testing.assert_equal, read_array, other_read_array
)

Xet Storage Details

Size:
16.2 kB
·
Xet hash:
fa44a6c3c0f87ae734b11dc3e46464f2da16287c2b028b194ee0034d9398b920

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.