Buckets:
| from pathlib import Path | |
| import numpy | |
| import pytest | |
| from PIL import Image | |
| from tiled.adapters.jpeg import JPEGAdapter, JPEGSequenceAdapter | |
| from tiled.adapters.mapping import MapAdapter | |
| from tiled.catalog import in_memory | |
| from tiled.client import Context, from_context | |
| from tiled.client.register import IMG_SEQUENCE_EMPTY_NAME_ROOT, register | |
| from tiled.server.app import build_app | |
| from tiled.utils import ensure_uri | |
| COLOR_SHAPE = (11, 17, 3) | |
| def client(tmpdir_module): | |
| sequence_directory = Path(tmpdir_module, "sequence") | |
| sequence_directory.mkdir() | |
| filepaths = [] | |
| for i in range(3): | |
| # JPEGs can only be 8 bit ints | |
| data = numpy.random.randint(0, 255, (5, 7), dtype="uint8") | |
| filepath = sequence_directory / f"temp{i:05}.jpeg" | |
| Image.fromarray(data).convert("L").save(filepath) | |
| filepaths.append(filepath) | |
| color_data = numpy.random.randint(0, 255, COLOR_SHAPE, dtype="uint8") | |
| path = Path(tmpdir_module, "color.jpeg") | |
| Image.fromarray(color_data).convert("RGB").save(path) | |
| tree = MapAdapter( | |
| { | |
| "color": JPEGAdapter.from_uris(ensure_uri(path)), | |
| "sequence": JPEGSequenceAdapter.from_uris( | |
| *[ensure_uri(filepath) for filepath in filepaths] | |
| ), | |
| } | |
| ) | |
| app = build_app(tree) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| yield client | |
| def test_jpeg_sequence(client, slice_input, correct_shape): | |
| arr = client["sequence"].read(slice=slice_input) | |
| assert arr.shape == correct_shape | |
| def test_jpeg_sequence_block(client, block_input, correct_shape): | |
| arr = client["sequence"].read_block(block_input) | |
| assert arr.shape == correct_shape | |
| async def test_jpeg_sequence_order(tmpdir): | |
| """ | |
| directory/ | |
| 00001.jpeg | |
| 00002.jpeg | |
| ... | |
| 00010.jpeg | |
| """ | |
| data = numpy.ones((4, 5)) | |
| num_files = 10 | |
| for i in range(num_files): | |
| Image.fromarray(data * i).convert("L").save(Path(tmpdir / f"image{i:05}.jpeg")) | |
| adapter = in_memory(readable_storage=[tmpdir]) | |
| with Context.from_app(build_app(adapter)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| for i in range(num_files): | |
| numpy.testing.assert_equal(client["image"][i], data * i) | |
| async def test_jpeg_sequence_with_directory_walker(tmpdir): | |
| """ | |
| directory/ | |
| 00001.jpeg | |
| 00002.jpeg | |
| ... | |
| 00010.jpeg | |
| single_image.jpeg | |
| image00001.jpeg | |
| image00002.jpeg | |
| ... | |
| image00010.jpeg | |
| other_image00001.jpeg | |
| other_image00002.jpeg | |
| ... | |
| other_image00010.jpeg | |
| other_image2_00001.jpeg | |
| other_image2_00002.jpeg | |
| ... | |
| other_image2_00010.jpeg | |
| other_file1.csv | |
| other_file2.csv | |
| stuff.csv | |
| """ | |
| data = numpy.random.randint(0, 255, (3, 5), dtype="uint8") | |
| for i in range(10): | |
| Image.fromarray(data).convert("L").save(Path(tmpdir / f"image{i:05}.jpeg")) | |
| Image.fromarray(data).convert("L").save( | |
| Path(tmpdir / f"other_image{i:05}.jpeg") | |
| ) | |
| Image.fromarray(data).convert("L").save(Path(tmpdir / f"{i:05}.jpeg")) | |
| Image.fromarray(data).convert("L").save( | |
| Path(tmpdir / f"other_image2_{i:05}.jpeg") | |
| ) | |
| Image.fromarray(data).save(Path(tmpdir / "single_image.jpeg")) | |
| for target in ["stuff.csv", "other_file1.csv", "other_file2.csv"]: | |
| with open(Path(tmpdir / target), "w") as file: | |
| file.write( | |
| """ | |
| a,b,c | |
| 1,2,3 | |
| """ | |
| ) | |
| adapter = in_memory(readable_storage=[tmpdir]) | |
| with Context.from_app(build_app(adapter)) as context: | |
| client = from_context(context) | |
| await register(client, tmpdir) | |
| # Single image is its own node. | |
| assert client["single_image"].shape == (3, 5) | |
| # Each sequence is grouped into a node. | |
| assert client[IMG_SEQUENCE_EMPTY_NAME_ROOT].shape == (10, 3, 5) | |
| assert client["image"].shape == (10, 3, 5) | |
| assert client["other_image"].shape == (10, 3, 5) | |
| assert client["other_image2_"].shape == (10, 3, 5) | |
| # The sequence grouping digit-only files appears with a uuid | |
| named_keys = [ | |
| "single_image", | |
| "image", | |
| "other_image", | |
| "other_image2_", | |
| "other_file1", | |
| "other_file2", | |
| "stuff", | |
| ] | |
| no_name_keys = [key for key in client.keys() if key not in named_keys] | |
| # There is only a single one of this type | |
| assert len(no_name_keys) == 1 | |
| assert client[no_name_keys[0]].shape == (10, 3, 5) | |
| # Other files are single nodes. | |
| assert client["stuff"].columns == ["a", "b", "c"] | |
| assert client["other_file1"].columns == ["a", "b", "c"] | |
| assert client["other_file2"].columns == ["a", "b", "c"] | |
| def test_rgb(client): | |
| "Test an RGB JPEG." | |
| arr = client["color"].read() | |
| assert arr.shape == COLOR_SHAPE | |
| def test_jpeg_sequence_cache(client): | |
| from numpy.testing import assert_raises | |
| # The two requests go through the same method in the server (read_block) to | |
| # call the same object | |
| indexed_array = client["sequence"][0] | |
| read_array = client["sequence"].read(0) | |
| # Using a different index to confirm that the previous cache doesn't affect the new array | |
| other_read_array = client["sequence"].read(1) | |
| numpy.testing.assert_equal(indexed_array, read_array) | |
| assert_raises( | |
| AssertionError, numpy.testing.assert_equal, read_array, other_read_array | |
| ) | |
Xet Storage Details
- Size:
- 6.17 kB
- Xet hash:
- b8cf59d4c1f64de1c4e2fffa035572e8759d28f9df0209b5f46e48c142f51787
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.