Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /bluesky_tiled /tests /test_container_fields.py
| import anyio | |
| import h5py | |
| import numpy as np | |
| import pandas | |
| import pytest | |
| import zarr | |
| from tiled.adapters.zarr import ZARR_LIB_V2 | |
| from tiled.catalog import in_memory | |
| from tiled.client import Context, from_context, record_history | |
| from tiled.client.register import register | |
| from tiled.examples.generate_files import generate_files | |
| from tiled.server.app import build_app | |
| def client(request: pytest.FixtureRequest): | |
| "A tiled client that serves from parametrized data_dir (Path or str)" | |
| data_dir = request.getfixturevalue(request.param) | |
| catalog = in_memory(readable_storage=[data_dir]) | |
| with Context.from_app(build_app(catalog)) as context: | |
| client = from_context(context) | |
| anyio.run(register, client, data_dir) | |
| yield client | |
| def example_data_dir(tmpdir_factory): | |
| "Generate a temporary directory with example files." | |
| tmpdir = tmpdir_factory.mktemp("example_files") | |
| generate_files(tmpdir) | |
| return tmpdir | |
| def test_directory_fields(client, fields, buffer): | |
| "Export selected fields (files) from a directory via /container/full." | |
| url_path = client.item["links"]["full"] | |
| with record_history() as history: | |
| client.export(buffer, fields=fields, format="application/x-hdf5") | |
| assert_single_request_to_url(history, url_path) | |
| assert_requested_fields_fetched(buffer, fields, client) | |
| def excel_data_dir(tmpdir_factory): | |
| "Generate a temporary Excel file with multiple Sheets of tabular data." | |
| tmpdir = tmpdir_factory.mktemp("excel_files") | |
| df = pandas.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) | |
| with pandas.ExcelWriter(tmpdir / "spreadsheet.xlsx") as writer: | |
| for i in range(10): | |
| df.to_excel(writer, sheet_name=f"Sheet {i+1}", index=False) | |
| return tmpdir | |
| def test_excel_fields(client, fields, buffer): | |
| "Export selected fields (sheets) from an Excel file via /container/full." | |
| client = client["spreadsheet"] | |
| url_path = client.item["links"]["full"] | |
| with record_history() as history: | |
| client.export(buffer, fields=fields, format="application/x-hdf5") | |
| # TODO: Enable container to export XLSX if all nodes are tables? | |
| # client.export(buffer, fields=fields, format=XLSX_MIME_TYPE) | |
| assert_single_request_to_url(history, url_path) | |
| assert_requested_fields_fetched(buffer, fields, client) | |
| def mark_xfail(value, unsupported="UNSPECIFIED ADAPTER"): | |
| "Indicate that this parameterized value is expected to fail" | |
| reason = f"Tiled does not currently support selecting 'fields' for {unsupported}" | |
| return pytest.param(value, marks=pytest.mark.xfail(reason=reason)) | |
| def zarr_data_dir(tmpdir_factory): | |
| "Generate a temporary Zarr group file with multiple datasets." | |
| tmpdir = tmpdir_factory.mktemp("zarr_files") | |
| try: | |
| root = zarr.open(str(tmpdir / "zarr_group.zarr"), mode="w") | |
| for i, name in enumerate("abcde"): | |
| if ZARR_LIB_V2: | |
| root.create_dataset(name, data=np.arange(i, i + 3)) | |
| else: | |
| root.create_array(name, data=np.arange(i, i + 3)) | |
| finally: | |
| # Ensure the Zarr group is closed properly | |
| print("Closed") | |
| return tmpdir | |
| def test_zarr_group_fields(client, fields, buffer): | |
| "Export selected fields (Datasets) from a Zarr group via /container/full." | |
| client = client["zarr_group"] | |
| # Normally, zarr would have 'attributes' stored as internal dictionary in the | |
| # metadata, but HDF5 does not support nested dictionaries. | |
| client.replace_metadata({"attributes": "", "zarr_format": 2 if ZARR_LIB_V2 else 3}) | |
| url_path = client.item["links"]["full"] | |
| with record_history() as history: | |
| client.export(buffer, fields=fields, format="application/x-hdf5") | |
| assert_single_request_to_url(history, url_path) | |
| assert_requested_fields_fetched(buffer, fields, client) | |
| def hdf5_data_dir(tmpdir_factory): | |
| "Generate a temporary HDF5 file with multiple entries." | |
| tmpdir = tmpdir_factory.mktemp("hdf5_files") | |
| with h5py.File(str(tmpdir / "hdf5_example.h5"), "w") as file: | |
| file["x"] = [1, 2, 3] | |
| group = file.create_group("g") | |
| group["y"] = [4, 5, 6] | |
| return tmpdir | |
| def test_hdf5_fields(client, fields, buffer): | |
| "Export selected fields (array/group) from a HDF5 file via /container/full." | |
| client = client["hdf5_example"] | |
| url_path = client.item["links"]["full"] | |
| with record_history() as history: | |
| client.export(buffer, fields=fields, format="application/x-hdf5") | |
| assert_single_request_to_url(history, url_path) | |
| assert_requested_fields_fetched(buffer, fields, client) | |
| def assert_single_request_to_url(history, url_path): | |
| "Container contents were fetched as a single request using 'full' route." | |
| (request,) = history.requests | |
| request_url_path = request.url.copy_with(query=None) | |
| assert request_url_path == url_path | |
| def assert_requested_fields_fetched(buffer, fields, client): | |
| "Only the requested fields were fetched." | |
| with h5py.File(buffer) as file: | |
| actual_fields = set(file.keys()) | |
| expected = set(fields or client.keys()) # By default all fields were fetched | |
| assert actual_fields == expected | |
Xet Storage Details
- Size:
- 5.95 kB
- Xet hash:
- 18a3ee2b65a23bb5ebaa72350b0157f6aecc30a3a90e97fef10cf9d88778aa09
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.