Buckets:
| import csv | |
| import json | |
| from pathlib import Path | |
| import numpy | |
| import pandas | |
| import pytest | |
| import xarray | |
| from tiled.adapters.array import ArrayAdapter | |
| from tiled.adapters.dataframe import DataFrameAdapter | |
| from tiled.adapters.mapping import MapAdapter | |
| from tiled.adapters.xarray import DatasetAdapter | |
| from tiled.client import Context, from_context | |
| from tiled.client.utils import ClientError | |
| from tiled.server.app import build_app | |
| data = numpy.random.random((10, 10)) | |
| temp = 15 + 8 * numpy.random.randn(2, 2, 3) | |
| precip = 10 * numpy.random.rand(2, 2, 3) | |
| lon = [[-99.83, -99.32], [-99.79, -99.23]] | |
| lat = [[42.25, 42.21], [42.63, 42.59]] | |
| tree = MapAdapter( | |
| { | |
| "A": ArrayAdapter.from_array(numpy.random.random((100, 100))), | |
| "B": ArrayAdapter.from_array(numpy.random.random((100, 100, 100))), | |
| "C": DataFrameAdapter.from_pandas( | |
| pandas.DataFrame( | |
| { | |
| "x": 1 * numpy.random.random(100), | |
| "y": 2 * numpy.random.random(100), | |
| "z": 3 * numpy.random.random(100), | |
| } | |
| ), | |
| npartitions=3, | |
| ), | |
| "empty_table": DataFrameAdapter.from_pandas( | |
| pandas.DataFrame({"A": []}), npartitions=1 | |
| ), | |
| "structured_data": MapAdapter( | |
| { | |
| "pets": ArrayAdapter.from_array( | |
| numpy.array( | |
| [("Rex", 9, 81.0), ("Fido", 3, 27.0)], | |
| dtype=[("name", "U10"), ("age", "i4"), ("weight", "f4")], | |
| ) | |
| ), | |
| "weather": DatasetAdapter.from_dataset( | |
| xarray.Dataset( | |
| { | |
| "temperature": (["x", "y", "time"], temp), | |
| "precipitation": (["x", "y", "time"], precip), | |
| }, | |
| coords={ | |
| "lon": (["x", "y"], lon), | |
| "lat": (["x", "y"], lat), | |
| "time": numpy.array( | |
| [1, 2, 3] | |
| ), # using ints here so HDF5 can export | |
| }, | |
| ) | |
| ), | |
| } | |
| ), | |
| } | |
| ) | |
| def client(): | |
| app = build_app(tree) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| yield client | |
| # We test a little bit of actual file export, using the tmpdir fixture, | |
| # but we mostly export for a buffer in memory because disk access | |
| # can be very cloud on cloud CI VMs. | |
| def has_csv_header(filepath): | |
| with open(filepath, "r") as csv_f: | |
| sniffer = csv.Sniffer() | |
| has_header = sniffer.has_header(csv_f.read(2048)) | |
| csv_f.seek(0) | |
| return has_header | |
| def test_export_2d_array(client, filename, tmpdir): | |
| client["A"].export(Path(tmpdir, filename)) | |
| def test_export_table(client, filename, tmpdir): | |
| client["C"].export(Path(tmpdir, filename)) | |
| def test_csv_mimetype_opt_params(client, filename, tmpdir): | |
| client["C"].export(Path(tmpdir, filename), format="text/csv;header=absent") | |
| assert not has_csv_header(Path(tmpdir, filename)) | |
| client["C"].export(Path(tmpdir, filename), format="text/csv;header=present") | |
| assert has_csv_header(Path(tmpdir, filename)) | |
| def test_streaming_export(client, buffer): | |
| "The application/json-seq format is streamed via a generator." | |
| client["C"].export(buffer, format="application/json-seq") | |
| # Verify that output is valid newline-delimited JSON. | |
| buffer.seek(0) | |
| lines = buffer.read().decode().splitlines() | |
| assert len(lines) == 100 | |
| for line in lines: | |
| json.loads(line) | |
| def test_streaming_export_empty(client, buffer): | |
| "The application/json-seq format is streamed via a generator." | |
| client["empty_table"].export(buffer, format="application/json-seq") | |
| buffer.seek(0) | |
| assert buffer.read() == b"" | |
| def test_export_weather_data_var(client, tmpdir, buffer): | |
| client["structured_data"]["weather"]["temperature"].export( | |
| buffer, slice=(0,), format="text/csv" | |
| ) | |
| def test_export_weather_all(client, buffer): | |
| client["structured_data"]["weather"].export(buffer, format="application/x-hdf5") | |
| def test_serialization_error_hdf5_metadata(client, buffer): | |
| tree = MapAdapter( | |
| { | |
| "good": MapAdapter({}, metadata={"a": 1}), | |
| "bad": MapAdapter({}, metadata={"a": {"b": 1}}), | |
| } | |
| ) | |
| with Context.from_app(build_app(tree)) as context: | |
| client = from_context(context) | |
| client["good"].export(buffer, format="application/x-hdf5") | |
| with pytest.raises(ClientError, match="contains types or structure"): | |
| client["bad"].export(buffer, format="application/x-hdf5") | |
| def test_path_as_Path_or_string(client, tmpdir): | |
| client["A"].export(Path(tmpdir, "test_path_as_path.txt")) | |
| client["A"].export(str(Path(tmpdir, "test_path_as_str.txt"))) | |
| def test_formats(client): | |
| client.formats | |
| client["A"].formats | |
| client["C"].formats | |
Xet Storage Details
- Size:
- 5.3 kB
- Xet hash:
- 2811c65dffc4e8deafb80c6be7f0d37999ce78571a9403526fba81fe4b4a4e60
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.