Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /bluesky_tiled /tests /test_awkward.py
| import json | |
| import awkward | |
| import numpy | |
| import pyarrow.feather | |
| import pyarrow.parquet | |
| import pytest | |
| from tiled.catalog import in_memory | |
| from tiled.client import Context, from_context, record_history | |
| from tiled.server.app import build_app | |
| from tiled.utils import APACHE_ARROW_FILE_MIME_TYPE | |
| from .utils import URL_LIMITS | |
| def catalog(tmpdir): | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| yield catalog | |
| def app(catalog): | |
| app = build_app(catalog) | |
| yield app | |
| def context(app): | |
| with Context.from_app(app) as context: | |
| yield context | |
| def client(context): | |
| client = from_context(context) | |
| yield client | |
| def test_slicing(client): | |
| # Write data into catalog. It will be stored as directory of buffers | |
| # named like 'node0-offsets' and 'node2-data'. | |
| array = awkward.Array( | |
| [ | |
| [{"x": 1.1, "y": [1]}, {"x": 2.2, "y": [1, 2]}], | |
| [], | |
| [{"x": 3.3, "y": [1, 2, 3]}], | |
| ] | |
| ) | |
| returned = client.write_awkward(array, key="test") | |
| # Test with client returned, and with client from lookup. | |
| for aac in [returned, client["test"]]: | |
| # Read the data back out from the AwkwardArrayClient, progressively sliced. | |
| assert awkward.almost_equal(aac.read(), array) | |
| assert awkward.almost_equal(aac[:], array) | |
| assert awkward.almost_equal(aac[0], array[0]) | |
| assert awkward.almost_equal(aac[0, "y"], array[0, "y"]) | |
| assert awkward.almost_equal(aac[0, "y", :1], array[0, "y", :1]) | |
| # When sliced, the serer sends less data. | |
| with record_history() as h: | |
| aac[:] | |
| assert len(h.responses) == 1 # sanity check | |
| full_response_size = len(h.responses[0].content) | |
| with record_history() as h: | |
| aac[0, "y"] | |
| assert len(h.responses) == 1 # sanity check | |
| sliced_response_size = len(h.responses[0].content) | |
| assert sliced_response_size < full_response_size | |
| def test_export_json(client, buffer): | |
| # Write data into catalog. It will be stored as directory of buffers | |
| # named like 'node0-offsets' and 'node2-data'. | |
| array = awkward.Array( | |
| [ | |
| [{"x": 1.1, "y": [1]}, {"x": 2.2, "y": [1, 2]}], | |
| [], | |
| [{"x": 3.3, "y": [1, 2, 3]}], | |
| ] | |
| ) | |
| aac = client.write_awkward(array, key="test") | |
| file = buffer | |
| aac.export(file, format="application/json") | |
| actual = bytes(file.getbuffer()).decode() | |
| assert actual == awkward.to_json(array) | |
| def test_export_arrow(tmpdir, client): | |
| # Write data into catalog. It will be stored as directory of buffers | |
| # named like 'node0-offsets' and 'node2-data'. | |
| array = awkward.Array( | |
| [ | |
| [{"x": 1.1, "y": [1]}, {"x": 2.2, "y": [1, 2]}], | |
| [], | |
| [{"x": 3.3, "y": [1, 2, 3]}], | |
| ] | |
| ) | |
| aac = client.write_awkward(array, key="test") | |
| filepath = tmpdir / "actual.arrow" | |
| aac.export(str(filepath), format=APACHE_ARROW_FILE_MIME_TYPE) | |
| actual = pyarrow.feather.read_table(filepath) | |
| expected = awkward.to_arrow_table(array) | |
| assert actual == expected | |
| def test_export_parquet(tmpdir, client): | |
| # Write data into catalog. It will be stored as directory of buffers | |
| # named like 'node0-offsets' and 'node2-data'. | |
| array = awkward.Array( | |
| [ | |
| [{"x": 1.1, "y": [1]}, {"x": 2.2, "y": [1, 2]}], | |
| [], | |
| [{"x": 3.3, "y": [1, 2, 3]}], | |
| ] | |
| ) | |
| aac = client.write_awkward(array, key="test") | |
| filepath = tmpdir / "actual.parquet" | |
| aac.export(str(filepath), format="application/x-parquet") | |
| actual = pyarrow.parquet.read_table(filepath) | |
| expected = awkward.to_arrow_table(array) | |
| assert actual == expected | |
| def test_large_number_of_form_keys(client): | |
| "Request should succeed even when too many form_keys to fit in a URL." | |
| # https://github.com/bluesky/tiled/pull/577 | |
| # The HTTP spec itself has no limit, but tools impose a pragmatic one. | |
| # This is a representative value. | |
| URL_LENGTH_LIMIT = URL_LIMITS.DEFAULT | |
| NUM_KEYS = 7_000 # meant to achieve a large body of form_keys | |
| array = awkward.Array([{f"key{i:05}": i for i in range(NUM_KEYS)}]) | |
| aac = client.write_awkward(array, key="test") | |
| with record_history() as h: | |
| aac[0] | |
| (request,) = h.requests | |
| assert request.method == "POST" | |
| form_keys = json.loads(request.read()) | |
| form_key_length = len(str(form_keys)) | |
| assert form_key_length > URL_LENGTH_LIMIT | |
| url_length = len(str(request.url)) | |
| assert url_length <= URL_LENGTH_LIMIT | |
| def test_more_slicing_1(client): | |
| array = awkward.Array( | |
| [ | |
| { | |
| "stuff": 123, | |
| "file": [ | |
| {"filename": 321, "other": 3.14}, | |
| ], | |
| }, | |
| ] | |
| ) | |
| returned = client.write_awkward(array, key="test") | |
| # Test with client returned, and with client from lookup. | |
| for aac in [returned, client["test"]]: | |
| # Read the data back out from the AwkwardArrayClient, progressively sliced. | |
| assert awkward.almost_equal(aac.read(), array) | |
| assert awkward.almost_equal(aac[:], array) | |
| assert awkward.almost_equal(aac["file", "filename"], array["file", "filename"]) | |
| def test_more_slicing_2(client): | |
| array = awkward.from_numpy( | |
| numpy.arange(2 * 3 * 5).reshape(2, 3, 5), regulararray=True | |
| ) | |
| returned = client.write_awkward(array, key="test") | |
| # Test with client returned, and with client from lookup. | |
| for aac in [returned, client["test"]]: | |
| # Read the data back out from the AwkwardArrayClient, progressively sliced. | |
| assert awkward.almost_equal(aac.read(), array) | |
| assert awkward.almost_equal(aac[:], array) | |
| assert awkward.almost_equal(aac[1:], array[1:]) | |
| assert awkward.almost_equal(aac[1:, 1:], array[1:, 1:]) | |
| def test_more_slicing_3(client): | |
| array = awkward.Array( | |
| [ | |
| {"good": 123, "bad": 123}, | |
| {"good": 321, "bad": [1, 2, 3]}, | |
| ] | |
| ) | |
| returned = client.write_awkward(array, key="test") | |
| # Test with client returned, and with client from lookup. | |
| for aac in [returned, client["test"]]: | |
| # Read the data back out from the AwkwardArrayClient, progressively sliced. | |
| assert awkward.almost_equal(aac.read(), array) | |
| assert awkward.almost_equal(aac[:], array[:]) | |
| assert awkward.almost_equal(aac["good"], array["good"]) | |
| record_arrays_examples = [ | |
| ( | |
| "simple_forms", | |
| [ | |
| {"a": 1, "b": "val_1", "c": 0.1, "d": True}, | |
| {"a": 2, "b": "val_2", "c": 0.2, "d": False}, | |
| {"a": 3, "b": "val_3", "c": 0.3, "d": True}, | |
| ], | |
| ), | |
| ( | |
| "nested_forms", | |
| [ | |
| {"a": [1, 2, 3], "b": {"x": 1, "y": 2}}, | |
| {"a": [4, 5, 6], "b": {"x": 1, "y": 2}}, | |
| {"a": [7, 8, 9], "b": {"x": 1, "y": 2}}, | |
| ], | |
| ), | |
| ( | |
| "empty_forms", | |
| [ | |
| {"a": [], "b": {}, "c": None, "d": 0.1}, | |
| {"a": [], "b": {}, "c": None}, | |
| {"a": [], "b": {}, "c": None}, | |
| ], | |
| ), | |
| ( | |
| "union_forms", | |
| [ | |
| {"a": 1, "b": True, "c": None}, | |
| {"a": 0.2, "b": [1, 2, 3]}, | |
| {"a": "three", "b": {"x": 1, "y": 2}}, | |
| ], | |
| ), | |
| ( | |
| "numpy_forms", | |
| [ | |
| {"a": numpy.ones(5), "b": numpy.empty(0), "c": numpy.zeros(1)}, | |
| {"a": numpy.ones(5), "b": numpy.empty(1), "c": numpy.zeros((2, 2))}, | |
| {"a": numpy.ones(5), "b": numpy.empty(2), "c": numpy.zeros((3, 3, 3))}, | |
| ], | |
| ), | |
| ] | |
| def test_record_arrays(client, name, data): | |
| array = awkward.Array(data) | |
| returned = client.write_awkward(array, key=f"test_{name}") | |
| # Test with client returned, and with client from lookup. | |
| for aac in [returned, client[f"test_{name}"]]: | |
| assert awkward.almost_equal(aac.read(), array) | |
| assert awkward.almost_equal(aac[:], array[:]) | |
| for field in array.fields: | |
| assert awkward.almost_equal(aac[field], array[field]) | |
Xet Storage Details
- Size:
- 8.21 kB
- Xet hash:
- 4e3c67711e36cbed62c06a587f68eac8e02ab08b822b48aab92b2031626b5a8a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.