Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /bluesky_tiled /tests /test_writing.py
"""
This tests tiled's writing routes with an in-memory store.
Persistent stores are being developed externally to the tiled package.
"""
import base64
import collections
import json
import math
import os
import pathlib
import threading
import uuid
from datetime import datetime
from urllib.parse import urljoin, urlparse
import awkward
import dask.dataframe
import numpy
import pandas
import pandas.testing
import pyarrow
import pytest
import sparse
from minio import Minio
from minio.error import S3Error
from pandas.testing import assert_frame_equal
from starlette.status import (
HTTP_404_NOT_FOUND,
HTTP_409_CONFLICT,
HTTP_415_UNSUPPORTED_MEDIA_TYPE,
HTTP_422_UNPROCESSABLE_CONTENT,
)
from tiled.catalog import in_memory
from tiled.catalog.adapter import CatalogContainerAdapter
from tiled.client import Context, from_context, record_history
from tiled.client.utils import ClientError
from tiled.mimetypes import PARQUET_MIMETYPE
from tiled.queries import Key
from tiled.server.app import build_app
from tiled.structures.array import BuiltinDtype
from tiled.structures.core import Spec, StructureFamily
from tiled.structures.data_source import DataSource
from tiled.structures.sparse import COOStructure
from tiled.structures.table import TableStructure
from tiled.utils import APACHE_ARROW_FILE_MIME_TYPE, patch_mimetypes, sanitize_uri
from tiled.validation_registration import ValidationRegistry
from .utils import fail_with_status_code
validation_registry = ValidationRegistry()
validation_registry.register("SomeSpec", lambda *args, **kwargs: None)
@pytest.fixture
def tmp_minio_bucket():
"""Create a temporary MinIO bucket and clean it up after tests."""
if uri := os.getenv("TILED_TEST_BUCKET"):
clean_uri, username, password = sanitize_uri(uri)
minio_client = Minio(
endpoint=urlparse(clean_uri).netloc, # e.g. only "localhost:9000"
access_key=username or "minioadmin",
secret_key=password or "minioadmin",
secure=False,
)
bucket_name = f"test-{uuid.uuid4().hex}"
minio_client.make_bucket(bucket_name=bucket_name)
try:
yield urljoin(uri, "/" + bucket_name) # full URI with credentials
finally:
# Cleanup: remove all objects and delete the bucket
try:
objects = minio_client.list_objects(
bucket_name=bucket_name, recursive=True
)
for obj in objects:
minio_client.remove_object(
bucket_name=bucket_name, object_name=obj.object_name
)
minio_client.remove_bucket(bucket_name=bucket_name)
except S3Error as e:
print(f"Warning: failed to delete test bucket {bucket_name}: {e}")
else:
yield None
@pytest.fixture
def tree(tmpdir, tmp_minio_bucket):
writable_storage = [f"duckdb:///{tmpdir / 'data.duckdb'}"]
if tmp_minio_bucket:
writable_storage.append(
{
"provider": "s3",
"uri": tmp_minio_bucket,
"config": {
"virtual_hosted_style_request": False,
"client_options": {"allow_http": True},
},
}
)
writable_storage.append(f"file://localhost{str(tmpdir / 'data')}")
return in_memory(writable_storage=writable_storage)
@pytest.fixture(scope="module")
def module_tmp_path(tmp_path_factory: pytest.TempdirFactory) -> pathlib.Path:
return tmp_path_factory.mktemp("temp")
@pytest.fixture(scope="module")
def metadata_client(module_tmp_path):
catalog = in_memory(writable_storage=str(module_tmp_path))
app = build_app(catalog)
with Context.from_app(app) as context:
client = from_context(context)
client.write_array([1, 2, 3], key="invalid_check")
yield client
def test_write_array_full(tree):
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
a = numpy.ones((5, 7))
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
with record_history() as history:
client.write_array(a, metadata=metadata, specs=specs)
# one request for metadata, one for data
assert len(history.requests) == 1 + 1
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_array = result.read()
numpy.testing.assert_equal(result_array, a)
assert result.metadata == metadata
assert result.specs == specs
def test_write_large_array_full(tree):
"Test that a large array is chunked"
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
a = numpy.ones(100, dtype=numpy.uint8)
# Low the limit so we can test on small data, for speed.
original = client._SUGGESTED_MAX_UPLOAD_SIZE
client._SUGGESTED_MAX_UPLOAD_SIZE = a.nbytes - 1
try:
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
with record_history() as history:
client.write_array(a, metadata=metadata, specs=specs)
# one request for metadata, more than one for data
assert len(history.requests) > 1 + 1
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_array = result.read()
numpy.testing.assert_equal(result_array, a)
assert result.metadata == metadata
assert result.specs == specs
finally:
client._SUGGESTED_MAX_UPLOAD_SIZE = original
@pytest.mark.parametrize(
"shape, chunks",
[((50, 30), (20, 15)), ((10, 1), (3, 1)), ((10, 1), (10, 1)), ((10,), (3,))],
)
def test_write_array_chunked(tree, shape, chunks):
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
a = dask.array.arange(math.prod(shape)).reshape(shape).rechunk(chunks)
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
with record_history() as history:
client.write_array(a, metadata=metadata, specs=specs)
# one request for metadata, multiple for data
assert len(history.requests) == 1 + a.npartitions
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_array = result.read()
numpy.testing.assert_equal(result_array, a.compute())
assert result.metadata == metadata
assert result.specs == specs
def test_extend_array(tree):
"Extend an array with additional data, expanding its shape."
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
a = numpy.ones((3, 2, 2))
new_data = numpy.ones((1, 2, 2)) * 2
full_array = numpy.concatenate((a, new_data), axis=0)
# Upload a (3, 2, 2) array.
ac = client.write_array(a)
assert ac.shape == a.shape
# Patching data into a region beyond the current extent of the array
# raises a ValueError (catching a 409 from the server).
with pytest.raises(ValueError):
ac.patch(new_data, offset=(3,))
# With extend=True, the array is expanded.
ac.patch(new_data, offset=(3,), extend=True)
# The local cache of the structure is updated.
assert ac.shape == full_array.shape
actual = ac.read()
# The array has the expected shape and data.
assert actual.shape == full_array.shape
numpy.testing.assert_equal(actual, full_array)
# Overwrite data (do not extend).
revised_data = numpy.ones((1, 2, 2)) * 3
revised_array = full_array.copy()
revised_array[3, :, :] = 3
ac.patch(revised_data, offset=(3,))
numpy.testing.assert_equal(ac.read(), revised_array)
# Extend out of order.
ones = numpy.ones((1, 2, 2))
ac.patch(ones * 7, offset=(7,), extend=True)
ac.patch(ones * 5, offset=(5,), extend=True)
ac.patch(ones * 6, offset=(6,), extend=True)
numpy.testing.assert_equal(ac[5:6], ones * 5)
numpy.testing.assert_equal(ac[6:7], ones * 6)
numpy.testing.assert_equal(ac[7:8], ones * 7)
# Offset given as an int is acceptable.
ac.patch(ones * 8, offset=8, extend=True)
numpy.testing.assert_equal(ac[8:9], ones * 8)
# Data type must match.
with pytest.raises(ValueError):
ac.patch(ones.astype("uint8"), offset=9, extend=True)
def test_write_table_full(tree):
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
data = {f"Column{i}": (1 + i) * numpy.ones(5) for i in range(5)}
df = pandas.DataFrame(data)
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
with record_history() as history:
client.write_table(df, metadata=metadata, specs=specs)
# one request for metadata, one for data
assert len(history.requests) == 1 + 1
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_dataframe = result.read()
pandas.testing.assert_frame_equal(result_dataframe, df)
assert result.metadata == metadata
assert result.specs == specs
def test_write_table_partitioned(tree):
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
data = {f"Column{i}": (1 + i) * numpy.ones(10) for i in range(5)}
df = pandas.DataFrame(data)
ddf = dask.dataframe.from_pandas(df, npartitions=3)
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
with record_history() as history:
client.write_table(ddf, metadata=metadata, specs=specs)
# one request for metadata, multiple for data
assert len(history.requests) == 1 + 3
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_dataframe = result.read()
pandas.testing.assert_frame_equal(result_dataframe, df)
assert result.metadata == metadata
assert result.specs == specs
def test_write_table_dict(tree):
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
data = {f"Column{i}": (1 + i) * numpy.ones(5) for i in range(5)}
df = pandas.DataFrame(data)
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
with record_history() as history:
client.write_table(data, metadata=metadata, specs=specs)
# one request for metadata, one for data
assert len(history.requests) == 1 + 1
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_dataframe = result.read()
pandas.testing.assert_frame_equal(result_dataframe, df)
assert result.metadata == metadata
assert result.specs == specs
@pytest.mark.parametrize(
"coo",
[
sparse.COO(
coords=numpy.array([[2, 5]]), data=numpy.array([1.3, 7.5]), shape=(10,)
),
sparse.COO(
coords=numpy.array([[0, 1], [2, 3]]),
data=numpy.array([3.8, 4.0]),
shape=(4, 4),
),
],
)
def test_write_sparse_full(tree, coo):
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
with record_history() as history:
client.write_sparse(
coords=coo.coords,
data=coo.data,
shape=coo.shape,
metadata=metadata,
specs=specs,
)
# one request for metadata, one for data
assert len(history.requests) == 1 + 1
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_array = result.read()
numpy.testing.assert_equal(result_array.todense(), coo.todense())
assert result.metadata == metadata
assert result.specs == specs
def test_write_sparse_chunked(tree):
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
metadata = {"scan_id": 1, "method": "A"}
specs = [Spec("SomeSpec")]
N = 5
with record_history() as history:
x = client.new(
"sparse",
[
DataSource(
structure=COOStructure(
shape=(2 * N,),
chunks=((N, N),),
data_type=BuiltinDtype.from_numpy_dtype(
numpy.dtype("float64")
),
),
structure_family="sparse",
)
],
metadata=metadata,
specs=specs,
)
x.write_block(coords=[[2, 4]], data=[3.1, 2.8], block=(0,))
x.write_block(coords=[[0, 1]], data=[6.7, 1.2], block=(1,))
# one request for metadata, multiple for data
assert len(history.requests) == 1 + 2
results = client.search(Key("scan_id") == 1)
result = results.values().first()
result_array = result.read()
assert numpy.array_equal(
result_array.todense(),
sparse.COO(
coords=numpy.array([[2, 4, N + 0, N + 1]]),
data=numpy.array([3.1, 2.8, 6.7, 1.2]),
shape=(10,),
).todense(),
)
# numpy.testing.assert_equal(result_array, sparse.COO(coords=[0, 1, ]))
assert result.metadata == metadata
assert result.specs == specs
def test_limits(tree):
"Test various limits on uploaded metadata."
MAX_ALLOWED_SPECS = 20
MAX_SPEC_CHARS = 255
validation_registry = ValidationRegistry()
for i in range(101):
validation_registry.register(f"spec{i}", lambda *args, **kwargs: None)
validation_registry.register("one_too_many", lambda *args, **kwargs: None)
validation_registry.register("a" * MAX_SPEC_CHARS, lambda *args, **kwargs: None)
validation_registry.register(
"a" * (1 + MAX_SPEC_CHARS), lambda *args, **kwargs: None
)
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
# Up to 20 specs are allowed.
max_allowed_specs = [f"spec{i}" for i in range(MAX_ALLOWED_SPECS)]
x = client.write_array([1, 2, 3], specs=max_allowed_specs)
x.update_metadata(specs=max_allowed_specs) # no-op
too_many_specs = max_allowed_specs + ["one_too_many"]
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
client.write_array([1, 2, 3], specs=too_many_specs)
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
x.update_metadata(specs=too_many_specs)
# Specs cannot repeat.
has_repeated_spec = ["spec0", "spec1", "spec0"]
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
client.write_array([1, 2, 3], specs=has_repeated_spec)
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
x.update_metadata(specs=has_repeated_spec)
# A given spec cannot be too long.
max_allowed_chars = ["a" * MAX_SPEC_CHARS]
client.write_array([1, 2, 3], specs=max_allowed_chars)
too_many_chars = ["a" * (1 + MAX_SPEC_CHARS)]
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
client.write_array([1, 2, 3], specs=too_many_chars)
with fail_with_status_code(HTTP_422_UNPROCESSABLE_CONTENT):
x.update_metadata(specs=too_many_chars)
def test_metadata_revisions(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
ac = client.write_array([1, 2, 3], key="revise_me")
assert len(ac.metadata_revisions[:]) == 0
ac.update_metadata(metadata={"a": 1})
assert ac.metadata["a"] == 1
assert client["revise_me"].metadata["a"] == 1
assert len(ac.metadata_revisions[:]) == 1
ac.update_metadata(metadata={"a": 2})
assert ac.metadata["a"] == 2
assert client["revise_me"].metadata["a"] == 2
assert len(ac.metadata_revisions[:]) == 2
ac.metadata_revisions.delete_revision(1)
assert len(ac.metadata_revisions[:]) == 1
with fail_with_status_code(HTTP_404_NOT_FOUND):
ac.metadata_revisions.delete_revision(1)
def test_metadata_revisions_count(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
ac = client.write_array([1, 2, 3], key="paginate_me")
ac.update_metadata(metadata={"a": 1})
ac.update_metadata(metadata={"a": 2})
ac.update_metadata(metadata={"a": 3})
# `len` reads the total count from the server, which was always 0 before.
assert len(ac.metadata_revisions) == 3
# Revisions are returned ordered by revision number.
revision_numbers = [r["revision_number"] for r in ac.metadata_revisions[:]]
assert revision_numbers == [1, 2, 3]
def test_metadata_revisions_count_empty(tree):
"""Cover the empty-page branch of `revisions_with_count`.
The windowed COUNT emits no rows when the SELECT is empty, so the
adapter falls back to a plain COUNT query. Exercise both empty
cases: (a) a node with zero revisions and (b) a page whose offset is
past the end of a non-empty revision list.
"""
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
# (a) Zero revisions on this node.
ac = client.write_array([1, 2, 3], key="no_revisions")
url = ac.uri.replace("/metadata/", "/revisions/")
page = context.http_client.get(url).json()
assert page["meta"]["count"] == 0
assert page["data"] == []
assert page["links"]["next"] is None
# (b) Offset past the end of a node that does have revisions.
ac = client.write_array([1, 2, 3], key="past_end")
ac.update_metadata(metadata={"a": 1})
ac.update_metadata(metadata={"a": 2})
url = ac.uri.replace("/metadata/", "/revisions/") + "?page[offset]=10"
page = context.http_client.get(url).json()
assert page["meta"]["count"] == 2
assert page["data"] == []
assert page["links"]["next"] is None
def test_metadata_revisions_pagination(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
ac = client.write_array([1, 2, 3], key="paginate_revisions")
ac.update_metadata(metadata={"a": 1})
ac.update_metadata(metadata={"a": 2})
ac.update_metadata(metadata={"a": 3})
# Walk one revision per page via the 'next' links (never generated before
# #1389) and confirm all three come back, in order, with no overlap.
url = ac.uri.replace("/metadata/", "/revisions/") + "?page[limit]=1"
revision_numbers = []
while url is not None:
page = context.http_client.get(url).json()
revision_numbers += [r["revision_number"] for r in page["data"]]
url = page["links"]["next"]
assert revision_numbers == [1, 2, 3]
def test_replace_metadata(tiled_websocket_context):
context = tiled_websocket_context
client = from_context(context)
unique_key = f"revise_me_with_replace_{uuid.uuid4().hex[:8]}"
# Write the initial data
ac = client.write_array([1, 2, 3], key=unique_key)
# Add Websocket Testing
# Set up subscription using the Subscription class with start=0
received = []
received_event = threading.Event()
def callback(update):
"""Callback to collect received messages."""
received.append(update)
if len(received) >= 3: # 3 updates
received_event.set()
# Create subscription for the streaming node with start=0
subscription = client.subscribe()
subscription.child_metadata_updated.add_callback(callback)
# Start the subscription
with subscription.start_in_thread(start=1):
# Business Logic
assert len(ac.metadata_revisions[:]) == 0
ac.replace_metadata(metadata={"a": 1}) # update #1
assert ac.metadata["a"] == 1
assert client[unique_key].metadata["a"] == 1
assert len(ac.metadata_revisions[:]) == 1
ac.replace_metadata(metadata={"a": 2}) # update #2
assert ac.metadata["a"] == 2
assert client[unique_key].metadata["a"] == 2
assert len(ac.metadata_revisions[:]) == 2
ac.metadata_revisions.delete_revision(1)
assert len(ac.metadata_revisions[:]) == 1
with fail_with_status_code(HTTP_404_NOT_FOUND):
ac.metadata_revisions.delete_revision(1)
ac.replace_metadata(metadata={"3": 1}, drop_revision=True) # update #3
# Wait for all messages to be received
assert received_event.wait(timeout=5.0), "Timeout waiting for messages"
@pytest.mark.parametrize(
"tested_metadata", ["test", ["test1", "test2"], 1, [1, 2], 1.2, True, (1, 2), {1}]
)
def test_invalid_metadata(metadata_client, tested_metadata):
with pytest.raises(ValueError):
metadata_client["invalid_check"].update_metadata(metadata=tested_metadata)
def test_valid_update_metadata(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
client.write_array([1, 2, 3], key="valid_check")
client["valid_check"].update_metadata(metadata={"a": 1, "b": 2})
assert client["valid_check"].metadata == {"a": 1, "b": 2}
d1 = {"a": 1}
d2 = {"b": 2}
d = collections.ChainMap(d1, d2)
client["valid_check"].update_metadata(metadata=d)
assert client["valid_check"].metadata == d
def test_drop_revision(tree):
key = "test_drop_revision"
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
# Set metadata color=blue.
ac = client.write_array([1, 2, 3], metadata={"color": "blue"}, key=key)
assert ac.metadata["color"] == "blue"
assert client[key].metadata["color"] == "blue"
assert len(ac.metadata_revisions[:]) == 0
# Update metadata to color=red, but drop revision.
ac.update_metadata(metadata={"color": "red"}, drop_revision=True)
# Metadata is updated; no revision is saved.
assert ac.metadata["color"] == "red"
assert client[key].metadata["color"] == "red"
assert len(ac.metadata_revisions) == 0
def test_merge_patching(tree):
"Test merge patching of metadata and specs"
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
ac = client.write_array([1, 2, 3], metadata={"a": 0, "b": 2}, specs=["spec1"])
ac.patch_metadata(
metadata_patch={"a": 1, "c": 3}, content_type=patch_mimetypes.MERGE_PATCH
)
assert dict(ac.metadata) == {"a": 1, "b": 2, "c": 3}
assert ac.specs[0].name == "spec1"
ac.patch_metadata(
specs_patch=["spec2"], content_type=patch_mimetypes.MERGE_PATCH
)
assert [x.name for x in ac.specs] == ["spec2"]
def test_json_patching(tree):
"Test json patching of metadata and specs"
validation_registry = ValidationRegistry()
for i in range(10):
validation_registry.register(f"spec{i}", lambda *args, **kwargs: None)
with Context.from_app(
build_app(tree, validation_registry=validation_registry)
) as context:
client = from_context(context)
ac = client.write_array([1, 2, 3], metadata={"a": 0, "b": 2}, specs=["spec1"])
ac.patch_metadata(
metadata_patch=[
{"op": "add", "path": "/c", "value": 3},
{"op": "replace", "path": "/a", "value": 1},
],
content_type=patch_mimetypes.JSON_PATCH,
)
assert dict(ac.metadata) == {"a": 1, "b": 2, "c": 3}
assert ac.specs[0].name == "spec1"
ac.patch_metadata(
specs_patch=[{"op": "add", "path": "/1", "value": "spec2"}],
content_type=patch_mimetypes.JSON_PATCH,
)
assert [x.name for x in ac.specs] == ["spec1", "spec2"]
def test_metadata_with_unsafe_objects(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
ac = client.write_array(
[1, 2, 3],
metadata={"date": datetime.now(), "array": numpy.array([1, 2, 3])},
key="x",
)
# Local copy matches copy fetched from remote.
assert ac.metadata == client["x"].metadata
@pytest.mark.asyncio
async def test_delete(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
client.write_array(
[1, 2, 3],
metadata={"date": datetime.now(), "array": numpy.array([1, 2, 3])},
key="delete_me",
)
nodes_before_delete = (await tree.context.execute("SELECT * from nodes")).all()
assert len(nodes_before_delete) == 1 + 1 # +1 for the root node
data_sources_before_delete = (
await tree.context.execute("SELECT * from data_sources")
).all()
assert len(data_sources_before_delete) == 1
assets_before_delete = (
await tree.context.execute("SELECT * from assets")
).all()
assert len(assets_before_delete) == 1
# Writing again with the same name fails.
with fail_with_status_code(HTTP_409_CONFLICT):
client.write_array(
[1, 2, 3],
metadata={"date": datetime.now(), "array": numpy.array([1, 2, 3])},
key="delete_me",
)
with pytest.raises(ClientError):
client.delete_contents("delete_me") # Cannot easily delete internal data
client.delete_contents("delete_me", external_only=False)
nodes_after_delete = (await tree.context.execute("SELECT * from nodes")).all()
assert len(nodes_after_delete) == 0 + 1 # the root node should still exist
data_sources_after_delete = (
await tree.context.execute("SELECT * from data_sources")
).all()
assert len(data_sources_after_delete) == 0
assets_after_delete = (await tree.context.execute("SELECT * from assets")).all()
assert len(assets_after_delete) == 0
# Writing again with the same name works now.
client.write_array(
[1, 2, 3],
metadata={"date": datetime.now(), "array": numpy.array([1, 2, 3])},
key="delete_me",
)
@pytest.mark.asyncio
async def test_delete_non_empty_node(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
a = client.create_container("a")
b = a.create_container("b")
c = b.create_container("c")
d = c.create_container("d")
# Cannot delete non-empty nodes
assert "a" in client
with fail_with_status_code(HTTP_409_CONFLICT):
client.delete_contents("a")
assert "b" in a
with fail_with_status_code(HTTP_409_CONFLICT):
a.delete_contents("b")
assert "c" in b
with fail_with_status_code(HTTP_409_CONFLICT):
b.delete_contents("c")
assert "d" in c
assert not list(d) # leaf is empty
# Delete from the bottom up.
c.delete_contents("d")
b.delete_contents("c")
a.delete_contents("b")
client.delete_contents("a")
@pytest.mark.asyncio
async def test_write_in_container(tree):
"Create a container and write a structure into it."
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
a = client.create_container("a")
df = pandas.DataFrame({"a": [1, 2, 3]})
b = a.write_table(df, key="b")
b.read()
a.delete_contents("b", external_only=False)
client.delete_contents("a")
a = client.create_container("a")
arr = numpy.array([1, 2, 3])
b = a.write_array(arr, key="b")
b.read()
a.delete_contents("b", external_only=False)
client.delete_contents("a")
a = client.create_container("a")
coo = sparse.COO(
coords=numpy.array([[2, 5]]), data=numpy.array([1.3, 7.5]), shape=(10,)
)
b = a.write_sparse(coords=coo.coords, data=coo.data, shape=coo.shape, key="b")
b.read()
a.delete_contents("b", external_only=False)
client.delete_contents("a")
a = client.create_container("a")
array = awkward.Array(
[
[{"x": 1.1, "y": [1]}, {"x": 2.2, "y": [1, 2]}],
[],
[{"x": 3.3, "y": [1, 2, 3]}],
]
)
b = a.write_awkward(array, key="b")
b.read()
a.delete_contents("b", external_only=False)
client.delete_contents("a")
@pytest.mark.asyncio
async def test_bytes_in_metadata(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
client.create_container("a", metadata={"test": b"raw_bytes"})
value = client["a"].metadata["test"]
assert value.startswith("data:application/octet-stream;base64,")
label, encoded = value.split(",", 1)
assert base64.b64decode(encoded) == b"raw_bytes"
@pytest.mark.asyncio
async def test_container_export(tree, buffer):
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
a = client.create_container("a")
a.write_array([1, 2, 3], key="b")
client.export(buffer, format="application/json")
@pytest.mark.parametrize("media_type", ["application/json", "application/json-seq"])
def test_table_bytes_column_json_export(tree, buffer, media_type):
"Binary columns are decoded to text so they can be JSON-serialized."
# b"\xff\xfe" is not valid UTF-8; it exercises the latin-1 fallback.
df = pandas.DataFrame(
{
"label": [b"abc", "café".encode(), b"\xff\xfe"],
"n": [1, 2, 3],
}
)
expected = ["abc", "café", b"\xff\xfe".decode("latin-1")]
with Context.from_app(build_app(tree)) as context:
client = from_context(context)
client.write_table(df, key="t")
client["t"].export(buffer, format=media_type)
payload = buffer.getvalue().decode("utf-8")
if media_type == "application/json":
assert json.loads(payload)["label"] == expected
else:
rows = [json.loads(line) for line in payload.splitlines()]
assert [row["label"] for row in rows] == expected
def test_write_with_specified_mimetype(tree):
with Context.from_app(build_app(tree)) as context:
client = from_context(context, include_data_sources=True)
df = pandas.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
structure = TableStructure.from_pandas(df)
for mimetype in [
PARQUET_MIMETYPE,
"text/csv",
APACHE_ARROW_FILE_MIME_TYPE,
]:
x = client.new(
"table",
[
DataSource(
structure_family=StructureFamily.table,
structure=structure,
mimetype=mimetype,
),
],
)
x.write_partition(0, df)
x.read()
x.refresh()
assert x.data_sources()[0].mimetype == mimetype
# Specifying unsupported mimetype raises expected error.
with fail_with_status_code(HTTP_415_UNSUPPORTED_MEDIA_TYPE):
client.new(
"table",
[
DataSource(
structure_family=StructureFamily.table,
structure=structure,
mimetype="application/x-does-not-exist",
),
],
)
@pytest.mark.parametrize(
"orig_file, file_to_append, expected_file",
[
(
{"A": [1, 2, 3], "B": [4, 5, 6]},
{"A": [11, 12, 13], "B": [14, 15, 16]},
{"A": [1, 2, 3, 11, 12, 13], "B": [4, 5, 6, 14, 15, 16]},
),
(
{"A": [1.2, 2.5, 3.7], "B": [4.6, 5.8, 6.9]},
{"A": [11.2, 12.5, 13.7], "B": [14.6, 15.8, 16.9]},
{
"A": [1.2, 2.5, 3.7, 11.2, 12.5, 13.7],
"B": [4.6, 5.8, 6.9, 14.6, 15.8, 16.9],
},
),
(
{"C": ["x", "y"], "D": ["a", "b"]},
{"C": ["xx", "yy", "zz"], "D": ["aa", "bb", "cc"]},
{"C": ["x", "y", "xx", "yy", "zz"], "D": ["a", "b", "aa", "bb", "cc"]},
),
],
)
def test_append_partition(
tree: CatalogContainerAdapter,
orig_file: dict,
file_to_append: dict,
expected_file: dict,
):
with Context.from_app(build_app(tree)) as context:
client = from_context(context, include_data_sources=True)
orig_table = pyarrow.Table.from_pydict(orig_file)
table_to_append = pyarrow.Table.from_pydict(file_to_append)
x = client.create_appendable_table(orig_table.schema, key="x")
x.append_partition(0, orig_table)
x.append_partition(0, table_to_append)
assert_frame_equal(x.read(), pandas.DataFrame(expected_file), check_dtype=False)
@pytest.mark.parametrize(
"table_name, expected",
[
(None, None),
("valid_table_name", None),
(
"_invalid_table_name",
pytest.raises(ValueError, match=r"Malformed SQL identifier.+"),
),
(
"invalid-table-name",
pytest.raises(ValueError, match=r"Malformed SQL identifier.+"),
),
(
"UPPERCASE_TABLE_NAME",
pytest.raises(ValueError, match=r"Malformed SQL identifier.+"),
),
(
"",
pytest.raises(ValueError, match=r"Malformed SQL identifier.+"),
),
],
)
def test_create_table_with_custom_name(
tree: CatalogContainerAdapter,
table_name: str,
expected: str,
):
table = pyarrow.Table.from_arrays([[1, 2, 3]], ["column_name"])
with Context.from_app(build_app(tree)) as context:
client = from_context(context, include_data_sources=True)
if isinstance(expected, type(pytest.raises(ValueError))):
with expected:
client.create_appendable_table(table.schema, table_name=table_name)
else:
x = client.create_appendable_table(table.schema, table_name=table_name)
x.append_partition(0, table)
assert x.read()["column_name"].to_list() == [1, 2, 3]
def test_deprecated_argument_order_raises_warning(tree: CatalogContainerAdapter):
table = pyarrow.Table.from_arrays([[1, 2, 3]], ["column_name"])
with Context.from_app(build_app(tree)) as context:
client = from_context(context, include_data_sources=True)
# Writable table
df = pandas.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
structure = TableStructure.from_pandas(df)
for mimetype in [
PARQUET_MIMETYPE,
"text/csv",
APACHE_ARROW_FILE_MIME_TYPE,
]:
x = client.new(
"table",
[
DataSource(
structure_family=StructureFamily.table,
structure=structure,
mimetype=mimetype,
),
],
)
with pytest.warns(DeprecationWarning, match=r"The order of arguments"):
x.write_partition(df, 0) # this argument order is deprecated
x.refresh()
assert x.read() is not None
# Appendable table
x = client.create_appendable_table(table.schema)
with pytest.warns(DeprecationWarning, match=r"The order of arguments"):
x.append_partition(table, 0) # this argument order is deprecated
assert x.read() is not None
"""
This tests tiled's writing routes with an in-memory store.
Persistent stores are being developed externally to the tiled package.
"""
import base64
import collections
import json
import math
import os
import pathlib
import threading
import uuid
from datetime import datetime
from urllib.parse import urljoin, urlparse
import awkward
import dask.dataframe
import numpy
import pandas
import pandas.testing
import pyarrow
import pytest
import sparse
from minio import Minio
from minio.error import S3Error
from pandas.testing import assert_frame_equal
from starlette.status import (
HTTP_404_NOT_FOUND,
HTTP_409_CONFLICT,
HTTP_415_UNSUPPORTED_MEDIA_TYPE,
HTTP_422_UNPROCESSABLE_CONTENT,
)
from tiled.catalog import in_memory
from tiled.catalog.adapter import CatalogContainerAdapter
from tiled.client import Context, from_context, record_history
from tiled.client.utils import ClientError
from tiled.mimetypes import PARQUET_MIMETYPE
from tiled.queries import Key
from tiled.server.app import build_app
from tiled.structures.array import BuiltinDtype
from tiled.structures.core import Spec, StructureFamily
from tiled.structures.data_source import DataSource
from tiled.structures.sparse import COOStructure
from tiled.structures.table import TableStructure
from tiled.utils import APACHE_ARROW_FILE_MIME_TYPE, patch_mimetypes, sanitize_uri
from tiled.validation_registration import ValidationRegistry
from .utils import fail_with_status_code
validation_registry = ValidationRegistry()
validation_registry.register("SomeSpec", lambda *args, **kwargs: None)
def tmp_minio_bucket():
"""Create a temporary MinIO bucket and clean it up after tests."""
if uri := os.getenv("TILED_TEST_BUCKET"):
clean_uri, username, password = sanitize_uri(uri)
minio_client = Minio(
endpoint=urlparse(clean_uri).netloc, # e.g. only "localhost:9000"
access_key=username or "minioadmin",
secret_key=password or "minioadmin",
secure=False,
)
bucket_name = f"test-{uuid.uuid4().hex}"
minio_client.make_bucket(bucket_name=bucket_name)
try:
yield urljoin(uri, "/" + bucket_name) # full URI with credentials
finally:
# Cleanup: remove all objects and delete the bucket
try:
objects = minio_client.list_objects(
bucket_name=bucket_name, recursive=True
)
for obj in objects:
minio_client.remove_object(
bucket_name=bucket_name, object_name=obj.object_name
)
minio_client.remove_bucket(bucket_name=bucket_name)
except S3Error as e:
print(f"Warning: failed to delete test bucket {bucket_name}: {e}")
else:
yield None
def tree(tmpdir, tmp_minio_bucket):
writable_storage = [f"duckdb:///{tmpdir / 'data.duckdb'}"]
if tmp_minio_bucket:
writable_storage.append(
{
"provider": "s3",
"uri": tmp_minio_bucket,
Xet Storage Details
- Size:
- 37.2 kB
- Xet hash:
- c142b32e990e92daf0f292af8c062362c620319aeb573f6152c19511db04297f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.