Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /bluesky_tiled /tests /test_directory_walker.py
| import dataclasses | |
| import platform | |
| import random | |
| from pathlib import Path | |
| import h5py | |
| import numpy | |
| import pytest | |
| import tifffile | |
| import yaml | |
| import zarr | |
| from starlette.status import HTTP_415_UNSUPPORTED_MEDIA_TYPE | |
| from tiled.adapters.hdf5 import HDF5Adapter | |
| from tiled.adapters.tiff import TiffAdapter | |
| from tiled.adapters.utils import init_adapter_from_catalog | |
| from tiled.catalog import in_memory | |
| from tiled.client import Context, from_context | |
| from tiled.client.register import ( | |
| Settings, | |
| group_image_sequences, | |
| identity, | |
| register, | |
| register_image_sequence, | |
| skip_all, | |
| strip_suffixes, | |
| ) | |
| from tiled.examples.generate_files import data, df1, generate_files | |
| from tiled.server.app import build_app | |
| from tiled.structures.array import ArrayStructure | |
| from tiled.structures.data_source import Asset, DataSource, Management | |
| from tiled.utils import ensure_uri, path_from_uri | |
| from .utils import fail_with_status_code | |
| def example_data_dir(tmpdir_factory): | |
| """ | |
| Generate a temporary directory with example files. | |
| The tmpdir_factory fixture ensures that this directory is cleaned up at test exit. | |
| """ | |
| tmpdir = tmpdir_factory.mktemp("example_files") | |
| generate_files(tmpdir) | |
| return tmpdir | |
| async def test_collision(example_data_dir, tmpdir): | |
| """Test that files which produce key collisions are ignored until the collision is resolved.""" | |
| # Add a.tiff which will collide with a.tif. | |
| p = Path(example_data_dir, "a.tiff") | |
| tifffile.imwrite(str(p), data) | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| await register(client, example_data_dir) | |
| # And omits the colliding entries. | |
| assert "a" not in client | |
| # Resolve the collision. | |
| p.unlink() | |
| # Re-run registration; entry should be there now. | |
| await register(client, example_data_dir) | |
| assert "a" in client | |
| def test_strip_suffixes(filename, expected): | |
| actual = strip_suffixes(filename) | |
| assert actual == expected | |
| async def test_same_filename_separate_directory(tmpdir): | |
| "Two files with the same name in separate directories should not collide." | |
| Path(tmpdir, "one").mkdir() | |
| Path(tmpdir, "two").mkdir() | |
| df1.to_csv(Path(tmpdir, "one", "a.csv")) | |
| df1.to_csv(Path(tmpdir, "two", "a.csv")) | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| assert "a" in client["one"] | |
| assert "a" in client["two"] | |
| async def test_register_prefix(tmpdir): | |
| "Register under a prefix" | |
| df1.to_csv(Path(tmpdir, "a.csv")) | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| x = client.create_container("x") | |
| y = x.create_container("y") | |
| y.create_container("z") # will be overwritten | |
| # Test that node is present. | |
| client["x", "y", "z"] | |
| await register(client, tmpdir, prefix="/x/y") | |
| # New node is present; old contents were overwritten. | |
| assert "a" in client["x/y"] | |
| assert "z" not in client["x/y"] | |
| async def test_mimetype_detection_hook(tmpdir): | |
| content = "a, b, c\n1, 2 ,3\n4, 5, 6\n" | |
| with open(Path(tmpdir / "a0"), "w") as file: | |
| file.write(content) | |
| with open(Path(tmpdir / "b0"), "w") as file: | |
| file.write(content) | |
| with open(Path(tmpdir / "c.csv"), "w") as file: | |
| file.write(content) | |
| with open(Path(tmpdir / "a.0.asfwoeijviojefeiofw"), "w") as file: | |
| file.write(content) | |
| with open(Path(tmpdir / "b.0.asfwoeijviojefeiofw"), "w") as file: | |
| file.write(content) | |
| def detect_mimetype(path, mimetype): | |
| filename = Path(path).name | |
| # If detection based on file extension worked, | |
| # we should get that in the mimetype. Otherwise, | |
| # mimetype should be None. | |
| if filename.endswith(".csv"): | |
| assert mimetype == "text/csv" | |
| else: | |
| assert mimetype is None | |
| if filename.startswith("a"): | |
| return "text/csv" | |
| return mimetype | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| await register( | |
| client, | |
| tmpdir, | |
| mimetype_detection_hook=detect_mimetype, | |
| key_from_filename=identity, | |
| ) | |
| assert set(client) == {"a0", "a.0.asfwoeijviojefeiofw", "c.csv"} | |
| async def test_skip_all_in_combination(tmpdir): | |
| "Using skip_all should override defaults, but not hinder other walkers" | |
| df1.to_csv(Path(tmpdir, "a.csv")) | |
| Path(tmpdir, "one").mkdir() | |
| df1.to_csv(Path(tmpdir, "one", "a.csv")) | |
| for i in range(2): | |
| tifffile.imwrite(Path(tmpdir, "one", f"image{i:05}.tif"), data) | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| # By default, both file and tiff sequence are registered. | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| assert "a" in client | |
| assert "a" in client["one"] | |
| assert "image" in client["one"] | |
| # With skip_all, directories and tiff sequence are registered, but individual files are not | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir, walkers=[group_image_sequences, skip_all]) | |
| assert list(client) == ["one"] | |
| assert "image" in client["one"] | |
| async def test_tiff_seq_custom_sorting(tmpdir): | |
| "Register images that are not in alphanumeric order." | |
| N = 10 | |
| ordering = list(range(N)) | |
| random.Random(0).shuffle(ordering) | |
| files = [] | |
| for i in ordering: | |
| file = Path(tmpdir, f"image{i:05}.tif") | |
| files.append(file) | |
| # data is a block of ones | |
| tifffile.imwrite(file, i * data) | |
| settings = Settings.init() | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| await register_image_sequence( | |
| client, | |
| "image", | |
| files, | |
| settings, | |
| ) | |
| # We are being a bit clever here. | |
| # Each image in this image series has pixels with a constant value, and | |
| # that value matches the image's position in the sequence enumerated by | |
| # `ordering`. We pick out one pixel and check that its value matches | |
| # the corresponding value in `ordering`. | |
| actual = list(client["image"][:, 0, 0]) | |
| assert actual == ordering | |
| async def test_image_file_with_sidecar_metadata_file(tmpdir): | |
| "Create one Node from two different types of files." | |
| MIMETYPE = "multipart/related;type=application/x-tiff-with-yaml" | |
| image_filepath = Path(tmpdir, "image.tif") | |
| tifffile.imwrite(image_filepath, data) | |
| metadata_filepath = Path(tmpdir, "metadata.yml") | |
| metadata = {"test_key": 3.0} | |
| with open(metadata_filepath, "w") as file: | |
| yaml.dump(metadata, file) | |
| class TiffAdapterWithSidecar(TiffAdapter): | |
| def __init__(self, image_uri, metadata_uri, metadata=None, **kwargs): | |
| with open(path_from_uri(metadata_uri)) as file: | |
| metadata = yaml.safe_load(file) | |
| super().__init__(image_uri, metadata=metadata, **kwargs) | |
| def from_catalog( | |
| cls, | |
| data_source, | |
| node, | |
| /, | |
| **kwargs, | |
| ): | |
| return init_adapter_from_catalog(cls, data_source, node, **kwargs) | |
| catalog = in_memory( | |
| writable_storage=str(tmpdir), | |
| adapters_by_mimetype={MIMETYPE: TiffAdapterWithSidecar}, | |
| ) | |
| with Context.from_app(build_app(catalog)) as context: | |
| adapter = TiffAdapterWithSidecar( | |
| ensure_uri(image_filepath), ensure_uri(metadata_filepath) | |
| ) | |
| client = from_context(context) | |
| client.new( | |
| key="image", | |
| structure_family=adapter.structure_family, | |
| metadata=dict(adapter.metadata()), | |
| specs=adapter.specs, | |
| data_sources=[ | |
| DataSource( | |
| structure_family=adapter.structure_family, | |
| mimetype=MIMETYPE, | |
| structure=dataclasses.asdict(adapter.structure()), | |
| parameters={}, | |
| management=Management.external, | |
| assets=[ | |
| Asset( | |
| data_uri=ensure_uri(metadata_filepath), | |
| is_directory=False, | |
| parameter="metadata_uri", | |
| ), | |
| Asset( | |
| data_uri=ensure_uri(image_filepath), | |
| is_directory=False, | |
| parameter="image_uri", | |
| ), | |
| ], | |
| ) | |
| ], | |
| ) | |
| assert numpy.array_equal(data, client["image"][:]) | |
| assert client["image"].metadata["test_key"] == 3.0 | |
| async def test_hdf5_virtual_datasets(tmpdir): | |
| # A virtual dataset comprises one master file and N data files. The master | |
| # file must be handed to the Adapter for opening. The data files are not | |
| # handled directly by the Adapter but they still ought to be tracked as | |
| # Assets for purposes of data movement, accounting for data size, etc. | |
| # This is why they are Assets with parameter=NULL/None, Assets not used | |
| # directly by the Adapter. | |
| # One could do one-dataset-per-directory. But like TIFF series in practice | |
| # they are often mixed, so we address that general case and track them at | |
| # the per-file level. | |
| # Contrast this to Zarr, where the files involves are always bundled by | |
| # directory. We track Zarr at the directory level. | |
| layout = h5py.VirtualLayout(shape=(4, 100), dtype="i4") | |
| data_filepaths = [] | |
| for n in range(1, 5): | |
| filepath = Path(tmpdir, f"{n}.h5") | |
| data_filepaths.append(filepath) | |
| vsource = h5py.VirtualSource(filepath, "data", shape=(100,)) | |
| layout[n - 1] = vsource | |
| # Add virtual dataset to output file | |
| filepath = Path(tmpdir, "VDS.h5") | |
| with h5py.File(filepath, "w", libver="latest") as file: | |
| file.create_virtual_dataset("data", layout, fillvalue=-5) | |
| assets = [ | |
| Asset( | |
| data_uri=str(ensure_uri(str(fp))), | |
| is_directory=False, | |
| parameter=None, # an indirect dependency | |
| ) | |
| for fp in data_filepaths | |
| ] | |
| assets.append( | |
| Asset( | |
| data_uri=ensure_uri(filepath), | |
| is_directory=False, | |
| parameter="data_uris", | |
| ) | |
| ) | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| adapter = HDF5Adapter.from_uris(ensure_uri(filepath)) | |
| client = from_context(context) | |
| client.new( | |
| key="VDS", | |
| structure_family=adapter.structure_family, | |
| metadata=dict(adapter.metadata()), | |
| specs=adapter.specs, | |
| data_sources=[ | |
| DataSource( | |
| structure_family=adapter.structure_family, | |
| mimetype="application/x-hdf5", | |
| structure=None, | |
| parameters={}, | |
| management=Management.external, | |
| assets=assets, | |
| ) | |
| ], | |
| ) | |
| client["VDS"]["data"][:] | |
| async def test_zarr_directory(tmpdir, register_parent): | |
| # Create a Zarr directory with some contents. | |
| zarr_dir = Path(tmpdir, "test.zarr") | |
| zarr_data = numpy.arange(200).reshape(10, 20) | |
| zarr_array = zarr.open( | |
| zarr_dir, mode="w", shape=(10, 20), chunks=(5, 5), dtype="i4" | |
| ) | |
| zarr_array[:] = zarr_data | |
| # Register either the Zarr directory itself, or its parent directory. | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir if register_parent else zarr_dir) | |
| assert list(client) == ["test"] | |
| assert client["test"].structure_family == "array" | |
| assert numpy.array_equal(client["test"][:], zarr_data) | |
| def test_unknown_mimetype(tmpdir): | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| asset = Asset( | |
| data_uri=ensure_uri(tmpdir / "test.does_not_exist"), | |
| is_directory=False, | |
| parameter="test", | |
| ) | |
| with fail_with_status_code(HTTP_415_UNSUPPORTED_MEDIA_TYPE): | |
| client.new( | |
| key="x", | |
| structure_family="array", | |
| metadata={}, | |
| specs=[], | |
| data_sources=[ | |
| DataSource( | |
| structure_family="array", | |
| mimetype="application/x-does-not-exist", | |
| structure=None, | |
| parameters={}, | |
| management=Management.external, | |
| assets=[asset], | |
| ) | |
| ], | |
| ) | |
| def test_one_asset_two_data_sources(tmpdir): | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| asset = Asset( | |
| data_uri=ensure_uri(tmpdir / "test.tiff"), | |
| is_directory=False, | |
| parameter="data_uri", | |
| num=None, | |
| ) | |
| for key in ["x", "y"]: | |
| client.new( | |
| key=key, | |
| structure_family="array", | |
| metadata={}, | |
| specs=[], | |
| data_sources=[ | |
| DataSource( | |
| structure_family="array", | |
| mimetype="image/tiff", | |
| structure=ArrayStructure.from_array(numpy.empty((5, 7))), | |
| parameters={}, | |
| management=Management.external, | |
| assets=[asset], | |
| ) | |
| ], | |
| ) | |
Xet Storage Details
- Size:
- 15 kB
- Xet hash:
- f6c83d07e3dac1aa904c1130edc019dd2bf3e383733789cc1dd2e392522b8a28
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.