Buckets:
| import enum | |
| import string | |
| from pathlib import Path | |
| import numpy | |
| import pandas | |
| import pytest | |
| from tiled.adapters.csv import CSVAdapter, CSVArrayAdapter | |
| from tiled.catalog import in_memory | |
| from tiled.client import Context, from_context | |
| from tiled.server.app import build_app | |
| from tiled.structures.array import ArrayStructure, Kind, StructDtype | |
| from tiled.structures.core import StructureFamily | |
| from tiled.structures.data_source import Asset, DataSource, Management | |
| from tiled.structures.table import TableStructure | |
| from tiled.utils import ensure_uri | |
| rng = numpy.random.default_rng(12345) | |
| df1 = pandas.DataFrame( | |
| { | |
| "A": ["red", "green", "blue", "white"], | |
| "B": [10.0, 20.0, 30.0, 40.0], | |
| "C": [0, 1, 2, 3], | |
| } | |
| ) | |
| arr1 = rng.integers(0, 255, size=(13, 17), dtype="uint8") | |
| arr2 = rng.random(size=(15, 19), dtype="float64") | |
| arr3 = rng.random(size=(15, 19), dtype="float32") | |
| df_arr1 = pandas.DataFrame(arr1) | |
| df_arr2 = pandas.DataFrame(arr2) | |
| df_arr3 = pandas.concat( | |
| [ | |
| pandas.DataFrame( | |
| rng.integers(0, 255, size=(10, 3), dtype="uint8"), columns=["A", "B", "C"] | |
| ), | |
| pandas.DataFrame( | |
| rng.random(size=(10, 3), dtype="float64"), columns=["D", "E", "F"] | |
| ), | |
| pandas.DataFrame( | |
| numpy.random.choice(list(string.ascii_letters), size=(10, 3)), | |
| columns=["G", "H", "I"], | |
| ), | |
| pandas.DataFrame( | |
| numpy.random.choice([True, False], size=(10, 3)), columns=["J", "K", "L"] | |
| ), | |
| ], | |
| axis=1, | |
| ) | |
| def tree(tmp_path_factory): | |
| return in_memory(writable_storage=tmp_path_factory.getbasetemp()) | |
| def context(tree): | |
| with Context.from_app(build_app(tree)) as context: | |
| client = from_context(context) | |
| client.create_container(key="x") | |
| yield context | |
| def csv_table_uri(tmpdir): | |
| fpath = Path(tmpdir, "table.csv") | |
| df1.to_csv(fpath, index=False) | |
| yield ensure_uri(fpath) | |
| def csv_array1_uri(tmpdir): | |
| fpath = Path(tmpdir, "array_1.csv") | |
| df_arr1.to_csv(fpath, index=False, header=False) | |
| yield ensure_uri(fpath) | |
| def csv_array2_uri(tmpdir): | |
| fpath = Path(tmpdir, "array_2.csv") | |
| df_arr2.to_csv(fpath, index=False, header=False) | |
| yield ensure_uri(fpath) | |
| def csv_array3_uri(tmpdir): | |
| fpath = Path(tmpdir, "array_3.csv") | |
| df_arr3.to_csv(fpath, index=False, header=True) | |
| yield ensure_uri(fpath) | |
| def test_csv_table(context, csv_table_uri): | |
| client = from_context(context) | |
| csv_assets = [ | |
| Asset( | |
| data_uri=csv_table_uri, | |
| is_directory=False, | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ] | |
| csv_data_source = DataSource( | |
| mimetype="text/csv;header=present", | |
| assets=csv_assets, | |
| structure_family=StructureFamily.table, | |
| structure=TableStructure.from_pandas(df1), | |
| management=Management.external, | |
| ) | |
| client["x"].new( | |
| structure_family=StructureFamily.table, | |
| data_sources=[csv_data_source], | |
| key="table", | |
| ) | |
| read_df = client["x"]["table"].read() | |
| assert set(read_df.columns) == set(df1.columns) | |
| assert (read_df == df1).all().all() | |
| def test_csv_table_from_uris(context, csv_table_uri, nullable): | |
| adp = CSVAdapter.from_uris(csv_table_uri, assume_missing=nullable) | |
| read_df = adp.read() | |
| assert set(read_df.columns) == set(df1.columns) | |
| assert (read_df == df1).all().all() | |
| assert read_df["C"].dtype == numpy.dtype("float64") if nullable else df1["C"].dtype | |
| def test_csv_struct_dtype_array(context, csv_table_uri): | |
| # Test reading a CSV table as a struct-dtyped array | |
| client = from_context(context) | |
| numpy_data_type = numpy.dtype([("A", "<U8"), ("B", "<f8"), ("C", "<i8")]) | |
| arr = df1.to_records(index=False).astype(numpy_data_type) | |
| structure = ArrayStructure( | |
| data_type=StructDtype.from_numpy_dtype(numpy_data_type), | |
| shape=(len(arr), 1), # the adapter should force the shape to be 2D | |
| chunks=((len(arr),), (1,)), | |
| ) | |
| csv_assets = [ | |
| Asset( | |
| data_uri=csv_table_uri, | |
| is_directory=False, | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ] | |
| csv_data_source = DataSource( | |
| mimetype="text/csv;header=absent", # ignore the header -- it is an "array" | |
| assets=csv_assets, | |
| parameters={"skiprows": 1, "header": None}, # skip the header row | |
| structure_family=StructureFamily.array, | |
| structure=structure, | |
| management=Management.external, | |
| ) | |
| client["x"].new( | |
| structure_family=StructureFamily.array, | |
| data_sources=[csv_data_source], | |
| key="struct_array", | |
| ) | |
| read_arr = client["x"]["struct_array"].read() | |
| assert read_arr.shape == (4, 1) | |
| assert (read_arr.ravel() == arr.ravel()).all() | |
| def test_csv_arrays(context, csv_array1_uri, csv_array2_uri): | |
| client = from_context(context) | |
| for key, data_uri, arr in zip( | |
| ("array1", "array2"), (csv_array1_uri, csv_array2_uri), (arr1, arr2) | |
| ): | |
| csv_assets = [ | |
| Asset( | |
| data_uri=data_uri, | |
| is_directory=False, | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ] | |
| csv_data_source = DataSource( | |
| mimetype="text/csv;header=absent", | |
| assets=csv_assets, | |
| structure_family=StructureFamily.array, | |
| structure=ArrayStructure.from_array(arr), | |
| management=Management.external, | |
| ) | |
| client["x"].new( | |
| structure_family=StructureFamily.array, | |
| data_sources=[csv_data_source], | |
| key=key, | |
| ) | |
| read_arr1 = client["x"]["array1"].read() | |
| assert numpy.array_equal(read_arr1, arr1) | |
| read_arr2 = client["x"]["array2"].read() | |
| assert numpy.isclose(read_arr2, arr2).all() | |
| def test_csv_arrays_selected_columns(context, csv_array3_uri, key, columns): | |
| client = from_context(context) | |
| orig_arr = df_arr3[columns].to_numpy() # The original array | |
| if "str" in key: | |
| orig_arr = orig_arr.astype("str") # Convert to string type explicitly | |
| csv_assets = [ | |
| Asset( | |
| data_uri=csv_array3_uri, | |
| is_directory=False, | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ] | |
| csv_data_source = DataSource( | |
| mimetype="text/csv;header=absent", | |
| assets=csv_assets, | |
| structure_family=StructureFamily.array, | |
| structure=ArrayStructure.from_array(orig_arr), | |
| management=Management.external, | |
| parameters={"header": 0, "usecols": columns}, | |
| ) | |
| client["x"].new( | |
| structure_family=StructureFamily.array, data_sources=[csv_data_source], key=key | |
| ) | |
| read_arr = client["x"][key].read() | |
| if "float" in key: | |
| assert numpy.isclose(read_arr, orig_arr).all() | |
| else: | |
| assert numpy.array_equal(read_arr, orig_arr) | |
| def test_csv_arrays_from_uris(csv_array1_uri, csv_array2_uri, csv_array3_uri, nullable): | |
| array_adapter = CSVArrayAdapter.from_uris(csv_array1_uri, assume_missing=nullable) | |
| read_arr = array_adapter.read() | |
| assert numpy.isclose(read_arr, arr1).all() | |
| assert read_arr.dtype == numpy.dtype("float64") if nullable else arr1.dtype | |
| array_adapter = CSVArrayAdapter.from_uris(csv_array2_uri, assume_missing=nullable) | |
| read_arr = array_adapter.read() | |
| assert numpy.isclose(read_arr, arr2).all() | |
| assert read_arr.dtype == numpy.dtype("float64") if nullable else arr2.dtype | |
| def test_csv_arrays_from_uris_selected_columns( | |
| csv_array3_uri, key, columns, monkeypatch, allow_object_arrays | |
| ): | |
| orig_arr = df_arr3[columns or df_arr3.columns] | |
| orig_arr = ( | |
| orig_arr.to_records(index=False).reshape(-1, 1) | |
| if "struct" in key | |
| else orig_arr.to_numpy() | |
| ) | |
| if allow_object_arrays: | |
| if "str" in key: | |
| # Allow object dtype to be used for string columns | |
| # This is equivalent to setting TILED_ALLOW_OBJECT_ARRAYS=1 | |
| kinds = {"object": "O", **{e.name: e.value for e in Kind}} | |
| NewKind = enum.Enum("Kind", kinds) | |
| monkeypatch.setattr("tiled.structures.array.Kind", NewKind) | |
| else: | |
| pytest.skip("Object arrays are only relevant for string columns") | |
| array_adapter = CSVArrayAdapter.from_uris(csv_array3_uri, header=0, usecols=columns) | |
| read_arr = array_adapter.read() | |
| if "struct" in key: | |
| for name in read_arr.dtype.names: | |
| a = read_arr[name] | |
| b = orig_arr[name] | |
| if numpy.issubdtype(a.dtype, numpy.floating): | |
| numpy.testing.assert_allclose(a, b) | |
| else: | |
| numpy.testing.assert_array_equal(a, b) | |
| elif ("float" in key) or ("numerical" in key): | |
| numpy.testing.assert_allclose(read_arr, orig_arr) | |
| else: | |
| numpy.testing.assert_array_equal(read_arr, orig_arr) | |
Xet Storage Details
- Size:
- 9.84 kB
- Xet hash:
- b0dc9615314887f03afaaaa0915cd3ea868e6c97fbda733006ba34c9ca3808ed
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.