Buckets:
| from pathlib import Path | |
| import numpy | |
| import pytest | |
| import tifffile as tf | |
| from tiled.adapters.mapping import MapAdapter | |
| from tiled.adapters.tiff import TiffAdapter, TiffSequenceAdapter | |
| from tiled.catalog import in_memory | |
| from tiled.client import Context, from_context | |
| from tiled.client.register import IMG_SEQUENCE_EMPTY_NAME_ROOT, register | |
| from tiled.server.app import build_app | |
| from tiled.structures.array import ArrayStructure, BuiltinDtype | |
| from tiled.utils import ensure_uri | |
| COLOR_SHAPE = (11, 17, 3) | |
| rng = numpy.random.default_rng(12345) | |
| tiff_data = rng.integers(0, 255, size=(12, 5, 7, 4), dtype="uint8") | |
| color_data = rng.integers(0, 255, size=COLOR_SHAPE, dtype="uint8") | |
| def client(tmpdir_module): | |
| sequence_directory = Path(tmpdir_module, "sequence") | |
| sequence_directory.mkdir() | |
| filepaths = [] | |
| for i in range(12): | |
| filepath = sequence_directory / f"temp{i:05}.tif" | |
| tf.imwrite(filepath, tiff_data[i, ...]) | |
| filepaths.append(filepath) | |
| path = Path(tmpdir_module, "color.tif") | |
| tf.imwrite(path, color_data) | |
| tree = MapAdapter( | |
| { | |
| "color": TiffAdapter(ensure_uri(path)), | |
| "sequence": TiffSequenceAdapter.from_uris( | |
| *[ensure_uri(filepath) for filepath in filepaths] | |
| ), | |
| # 5d sequence stacked along the leftmost dimension | |
| "5d_sequence": TiffSequenceAdapter( | |
| [ensure_uri(filepath) for filepath in filepaths[:3]], | |
| structure=ArrayStructure( | |
| shape=(3, 1, 5, 7, 4), | |
| chunks=((1, 1, 1), (1,), (5,), (7,), (4,)), | |
| data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")), | |
| ), | |
| ), | |
| # 5d sequence stacked along the second dimension | |
| "5d_sequence_second_dim": TiffSequenceAdapter( | |
| [ensure_uri(filepath) for filepath in filepaths[:3]], | |
| structure=ArrayStructure( | |
| shape=(1, 3, 5, 7, 4), | |
| chunks=((1,), (1,) * 3, (5,), (7,), (4,)), | |
| data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")), | |
| ), | |
| ), | |
| # 5d sequence stacked along the first and second dimensions | |
| "5d_sequence_first_and_second_dim": TiffSequenceAdapter( | |
| [ensure_uri(filepath) for filepath in filepaths], | |
| structure=ArrayStructure( | |
| shape=(3, 4, 5, 7, 4), | |
| chunks=((1, 1, 1), (2, 2), (5,), (7,), (4,)), | |
| data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")), | |
| ), | |
| ), | |
| } | |
| ) | |
| # `_chunks` would be provided in data_source.properties if initialized from a catalog | |
| tree["5d_sequence"]._chunks = ((1, 1, 1), (5,), (7,), (4,)) | |
| tree["5d_sequence_second_dim"]._chunks = ((1, 1, 1), (5,), (7,), (4,)) | |
| tree["5d_sequence_first_and_second_dim"]._chunks = ((1,) * 12, (5,), (7,), (4,)) | |
| app = build_app(tree) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| yield client | |
| def test_tiff_sequence(client, slice_input, correct_shape): | |
| arr = client["sequence"].read(slice=slice_input) | |
| assert arr.shape == correct_shape | |
| true_arr = tiff_data[slice_input or ()] | |
| numpy.testing.assert_equal(arr, true_arr) | |
| def test_forced_reshaping(client, slice_input, correct_shape): | |
| arr = client["5d_sequence"].read(slice=slice_input) | |
| assert arr.shape == correct_shape | |
| slice_input = () if slice_input is None else slice_input | |
| true_arr = tiff_data[:3, ...].reshape(3, 1, 5, 7, 4)[slice_input] | |
| numpy.testing.assert_equal(arr, true_arr) | |
| def test_forced_reshaping_along_second_dimension(client, slice_input, correct_shape): | |
| """Test that the presence of a frame per point dimension does not affect | |
| the ability to read with a slice that would normally cause reshaping. | |
| """ | |
| arr = client["5d_sequence_second_dim"].read(slice=slice_input) | |
| assert arr.shape == correct_shape | |
| slice_input = () if slice_input is None else slice_input | |
| true_arr = tiff_data[:3, ...].reshape(1, 3, 5, 7, 4)[slice_input] | |
| numpy.testing.assert_equal(arr, true_arr) | |
| def test_forced_reshaping_along_first_and_second_dimensions( | |
| client, slice_input, correct_shape | |
| ): | |
| """Test that the presence of a frame per point dimension does not affect | |
| the ability to read with a slice that would normally cause reshaping. | |
| """ | |
| arr = client["5d_sequence_first_and_second_dim"].read(slice=slice_input) | |
| assert arr.shape == correct_shape | |
| slice_input = () if slice_input is None else slice_input | |
| true_arr = tiff_data.reshape(3, 4, 5, 7, 4)[slice_input] | |
| numpy.testing.assert_equal(arr, true_arr) | |
| def test_tiff_sequence_block(client, block_input, correct_shape): | |
| arr = client["sequence"].read_block(block_input) | |
| assert arr.shape == correct_shape | |
| def test_reshaped_sequence_reducing_slice_is_batch_invariant( | |
| client, monkeypatch, slice_input, correct_shape | |
| ): | |
| """A slice that touches many files but keeps only part of each frame must be | |
| read in memory-bounded batches without changing the result. Force a tiny | |
| per-batch budget so every file lands in its own batch, and check the result | |
| matches both numpy ground truth and a single-batch read. | |
| """ | |
| import tiled.adapters.sequence as sequence_module | |
| true_arr = tiff_data.reshape(3, 4, 5, 7, 4)[slice_input] | |
| # Single batch (generous budget). | |
| monkeypatch.setattr(sequence_module, "READ_BATCH_BYTES", 1 << 30) | |
| single = client["5d_sequence_first_and_second_dim"].read(slice=slice_input) | |
| assert single.shape == correct_shape | |
| numpy.testing.assert_equal(single, true_arr) | |
| # Force one file per batch; result must be identical. | |
| monkeypatch.setattr(sequence_module, "READ_BATCH_BYTES", 1) | |
| batched = client["5d_sequence_first_and_second_dim"].read(slice=slice_input) | |
| assert batched.shape == correct_shape | |
| numpy.testing.assert_equal(batched, true_arr) | |
| async def test_tiff_sequence_order(tmpdir): | |
| """ | |
| directory/ | |
| 00001.tif | |
| 00002.tif | |
| ... | |
| 00010.tif | |
| """ | |
| data = numpy.ones((4, 5)) | |
| num_files = 10 | |
| for i in range(num_files): | |
| tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), data * i) | |
| adapter = in_memory(readable_storage=[tmpdir]) | |
| with Context.from_app(build_app(adapter)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| for i in range(num_files): | |
| numpy.testing.assert_equal(client["image"][i], data * i) | |
| async def test_tiff_sequence_with_directory_walker(tmpdir): | |
| """ | |
| directory/ | |
| 00001.tif | |
| 00002.tif | |
| ... | |
| 00010.tif | |
| single_image.tif | |
| image00001.tif | |
| image00002.tif | |
| ... | |
| image00010.tif | |
| other_image00001.tif | |
| other_image00002.tif | |
| ... | |
| other_image00010.tif | |
| other_image2_00001.tif | |
| other_image2_00002.tif | |
| ... | |
| other_image2_00010.tif | |
| other_file1.csv | |
| other_file2.csv | |
| stuff.csv | |
| """ | |
| data = numpy.random.random((3, 5)) | |
| for i in range(10): | |
| tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), data) | |
| tf.imwrite(Path(tmpdir / f"other_image{i:05}.tif"), data) | |
| tf.imwrite(Path(tmpdir / f"{i:05}.tif"), data) | |
| tf.imwrite(Path(tmpdir / f"other_image2_{i:05}.tif"), data) | |
| tf.imwrite(Path(tmpdir / "single_image.tif"), data) | |
| for target in ["stuff.csv", "other_file1.csv", "other_file2.csv"]: | |
| with open(Path(tmpdir / target), "w") as file: | |
| file.write( | |
| """ | |
| a,b,c | |
| 1,2,3 | |
| """ | |
| ) | |
| adapter = in_memory(readable_storage=[tmpdir]) | |
| with Context.from_app(build_app(adapter)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| # Single image is its own node. | |
| assert client["single_image"].shape == (3, 5) | |
| # Each sequence is grouped into a node. | |
| assert client[IMG_SEQUENCE_EMPTY_NAME_ROOT].shape == (10, 3, 5) | |
| assert client["image"].shape == (10, 3, 5) | |
| assert client["other_image"].shape == (10, 3, 5) | |
| assert client["other_image2_"].shape == (10, 3, 5) | |
| # The sequence grouping digit-only files appears with a uuid | |
| named_keys = [ | |
| "single_image", | |
| "image", | |
| "other_image", | |
| "other_image2_", | |
| "other_file1", | |
| "other_file2", | |
| "stuff", | |
| ] | |
| no_name_keys = [key for key in client.keys() if key not in named_keys] | |
| # There is only a single one of this type | |
| assert len(no_name_keys) == 1 | |
| assert client[no_name_keys[0]].shape == (10, 3, 5) | |
| # Other files are single nodes. | |
| assert client["stuff"].columns == ["a", "b", "c"] | |
| assert client["other_file1"].columns == ["a", "b", "c"] | |
| assert client["other_file2"].columns == ["a", "b", "c"] | |
| async def test_tiff_sequence_client_data_sources_assets(tmpdir): | |
| """A file-sequence node exposes ALL its assets to the client via | |
| `c.data_sources()[0].assets`, even though the server loads assets | |
| lazily (asset-free `data_sources` property + on-demand per-frame reads). | |
| Also confirms single-frame reads return correct data, exercising the lazy | |
| per-frame resolution path end-to-end through the catalog. | |
| """ | |
| num_frames = 10 | |
| base = numpy.random.default_rng(0).integers(0, 255, size=(3, 5), dtype="uint8") | |
| frames = [] | |
| for i in range(num_frames): | |
| arr = (base + i).astype("uint8") | |
| tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), arr) | |
| frames.append(arr) | |
| adapter = in_memory(readable_storage=[tmpdir]) | |
| with Context.from_app(build_app(adapter)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| seq = client["image"] | |
| assert seq.shape == (num_frames, 3, 5) | |
| # Client-facing data_sources() must still expose every asset. | |
| data_sources = seq.data_sources() | |
| assert data_sources is not None | |
| assert len(data_sources) == 1 | |
| assets = data_sources[0].assets | |
| assert len(assets) == num_frames | |
| assert all(asset.data_uri for asset in assets) | |
| # Single-frame reads via the lazy path return the correct frame. | |
| for i in (0, 3, num_frames - 1): | |
| numpy.testing.assert_array_equal(seq[i], frames[i]) | |
| # A multi-frame slice also resolves correctly. | |
| numpy.testing.assert_array_equal(seq[2:5], numpy.stack(frames[2:5])) | |
| async def test_tiff_sequence_lazy_http_block_and_slice_reads(tmpdir): | |
| """End-to-end HTTP exercise of the lazy per-frame path. | |
| Reads reach the server via the `/array/full` (slice) and `/array/block` | |
| routes, which dispatch to the catalog's lazy resolver. Confirms both give | |
| data identical to the eager reference for whole-array, sliced, strided, and | |
| single-block reads. | |
| """ | |
| num_frames = 6 | |
| base = numpy.random.default_rng(1).integers(0, 255, size=(3, 5), dtype="uint8") | |
| frames = [] | |
| for i in range(num_frames): | |
| arr = (base + i).astype("uint8") | |
| tf.imwrite(Path(tmpdir / f"image{i:05}.tif"), arr) | |
| frames.append(arr) | |
| stacked = numpy.stack(frames) | |
| adapter = in_memory(readable_storage=[tmpdir]) | |
| with Context.from_app(build_app(adapter)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| seq = client["image"] | |
| assert seq.shape == (num_frames, 3, 5) | |
| # Full read over HTTP (/array/full). | |
| numpy.testing.assert_array_equal(seq.read(), stacked) | |
| # Contiguous and strided slice reads over HTTP. | |
| numpy.testing.assert_array_equal(seq.read(slice=(slice(1, 4),)), stacked[1:4]) | |
| numpy.testing.assert_array_equal( | |
| seq.read(slice=(slice(0, num_frames, 2),)), stacked[0:num_frames:2] | |
| ) | |
| # Per-block reads over HTTP (/array/block): each file is one block on axis 0. | |
| for i in (0, 2, num_frames - 1): | |
| block = seq.read_block((i, 0, 0)) | |
| numpy.testing.assert_array_equal(block, frames[i][numpy.newaxis]) | |
| # Block read with a within-block slice (/array/block?slice=...): the block | |
| # selects the file(s) to resolve; the within-block slice is applied to the | |
| # already-loaded block data and must never change which files are read. | |
| for i in (0, 3, num_frames - 1): | |
| block = seq.read_block((i, 0, 0), slice=(0, slice(1, 3))) | |
| numpy.testing.assert_array_equal(block, frames[i][1:3]) | |
| def test_rgb(client): | |
| "Test an RGB TIFF." | |
| arr = client["color"].read() | |
| assert arr.shape == COLOR_SHAPE | |
| def test_tiff_sequence_cache(client): | |
| from numpy.testing import assert_raises | |
| # The two requests go through the same method in the server (read_block) to | |
| # call the same object | |
| indexed_array = client["sequence"][0] | |
| read_array = client["sequence"].read(0) | |
| # Using a different index to confirm that the previous cache doesn't affect the new array | |
| other_read_array = client["sequence"].read(1) | |
| numpy.testing.assert_equal(indexed_array, read_array) | |
| assert_raises( | |
| AssertionError, numpy.testing.assert_equal, read_array, other_read_array | |
| ) | |
Xet Storage Details
- Size:
- 16.2 kB
- Xet hash:
- fa44a6c3c0f87ae734b11dc3e46464f2da16287c2b028b194ee0034d9398b920
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.