Buckets:
| """Integration tests for the `bytes` structure family. | |
| The `bytes` family has no dedicated content endpoint and no family-specific | |
| client. Clients register an opaque payload, then download each underlying | |
| asset through `/asset/bytes/{path}?id=N`, gated by `settings.expose_raw_assets`. | |
| These tests exercise that full round-trip end-to-end. | |
| """ | |
| import os | |
| import warnings | |
| from pathlib import Path | |
| from unittest import mock | |
| import pytest | |
| from rich.progress import Progress | |
| from tiled.adapters.bytes import BytesAdapter | |
| from tiled.catalog import in_memory | |
| from tiled.client import Context, from_context, record_history | |
| from tiled.client.bytes import BytesClient | |
| from tiled.client.download import STREAMING_ACCEPT_ENCODING, download | |
| from tiled.server.app import build_app | |
| from tiled.structures.bytes import BytesStructure | |
| from tiled.structures.core import StructureFamily | |
| from tiled.structures.data_source import Asset, DataSource, Management | |
| PAYLOAD = b"the quick brown fox jumps over the lazy dog" | |
| def http_client(tmpdir): | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| with Context.from_app(build_app(catalog)) as ctx: | |
| yield from_context(ctx) | |
| def _register_bytes_node( | |
| client, | |
| tmp_path, | |
| payload, | |
| key="blob", | |
| mimetype="application/octet-stream", | |
| size="auto", | |
| ): | |
| """Write `payload` to a file and register it as an external bytes node. | |
| `size` defaults to the payload length; pass `None` to register an asset | |
| whose size is unknown (mimicking a legacy row).""" | |
| p = tmp_path / f"{key}.bin" | |
| p.write_bytes(payload) | |
| data_source = DataSource( | |
| mimetype=mimetype, | |
| assets=[ | |
| Asset( | |
| data_uri=p.as_uri(), | |
| is_directory=False, | |
| size=len(payload) if size == "auto" else size, | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ], | |
| structure_family=StructureFamily.bytes, | |
| structure=BytesStructure(), | |
| management=Management.external, | |
| ) | |
| return client.new( | |
| structure_family=StructureFamily.bytes, | |
| data_sources=[data_source], | |
| key=key, | |
| ) | |
| def _register_multi_asset_node(client, tmp_path, chunks, key="multi", reverse=False): | |
| """Write each chunk to its own file and register them as one bytes node. | |
| `num` is set to the chunk's index, so the original payload can be recovered | |
| by ordering downloaded parts by `num`. When `reverse=True`, the assets are | |
| submitted in the *opposite* order to verify the server canonicalises by `num`. | |
| """ | |
| assets = [] | |
| for i, chunk in enumerate(chunks): | |
| p = tmp_path / f"c{i:02d}.bin" | |
| p.write_bytes(chunk) | |
| assets.append( | |
| Asset( | |
| data_uri=p.as_uri(), | |
| is_directory=False, | |
| size=len(chunk), | |
| parameter="data_uris", | |
| num=i, | |
| ) | |
| ) | |
| client.new( | |
| structure_family=StructureFamily.bytes, | |
| data_sources=[ | |
| DataSource( | |
| mimetype="application/octet-stream", | |
| assets=list(reversed(assets)) if reverse else assets, | |
| structure_family=StructureFamily.bytes, | |
| structure=BytesStructure(), | |
| management=Management.external, | |
| ) | |
| ], | |
| key=key, | |
| ) | |
| class _DiskDestination: | |
| """`raw_export` target backed by a directory on disk.""" | |
| def __init__(self, tmp_path): | |
| self.dest = tmp_path / "out" | |
| self.dest.mkdir() | |
| def arg(self): | |
| return self.dest | |
| def read(self, handle): | |
| return Path(handle).read_bytes() | |
| def asset_id(self, handle): | |
| # Multi-asset disk layout: `<dest>/<asset_id>/<filename>`. | |
| return int(Path(handle).parent.name) | |
| class _MappingDestination: | |
| """`raw_export` target backed by an in-memory `dict` of `BytesIO` buffers.""" | |
| def __init__(self, tmp_path): | |
| self.buffers = {} | |
| def arg(self): | |
| return self.buffers | |
| def read(self, handle): | |
| return self.buffers[handle].read() | |
| def asset_id(self, handle): | |
| # Multi-asset mapping layout: `<asset_id>/<filename>`. | |
| return int(handle.split("/")[0]) | |
| def destination(request, tmp_path): | |
| return {"disk": _DiskDestination, "mapping": _MappingDestination}[request.param]( | |
| tmp_path | |
| ) | |
| def test_roundtrip_register_and_download(http_client, tmp_path): | |
| """Register a bytes node, inspect its metadata, and download the asset. | |
| Covers the full path: structure_family wiring, default mimetype, | |
| Asset.size population, asset-id discovery, and content round-trip via /asset/bytes. | |
| Also exercises BytesClient dispatch and its size-aware __repr__. | |
| """ | |
| handle = _register_bytes_node(http_client, tmp_path, PAYLOAD, key="blob") | |
| assert handle.item["attributes"]["structure_family"] == "bytes" | |
| # Dispatch resolves bytes nodes to BytesClient, not generic BaseClient. | |
| assert isinstance(http_client["blob"], BytesClient) | |
| # Without include_data_sources(), __repr__ falls back to the class name | |
| # and must not trigger a network round-trip. | |
| assert repr(http_client["blob"]) == "<BytesClient>" | |
| # With data sources cached, __repr__ reports total size and asset count. | |
| assert ( | |
| repr(http_client["blob"].include_data_sources()) | |
| == f"<BytesClient {len(PAYLOAD)} B across 1 asset>" | |
| ) | |
| meta = http_client.context.http_client.get( | |
| "/api/v1/metadata/blob", params={"include_data_sources": True} | |
| ) | |
| ds = meta.json()["data"]["attributes"]["data_sources"][0] | |
| assert ds["mimetype"] == "application/octet-stream" | |
| asset = ds["assets"][0] | |
| assert asset["size"] == len(PAYLOAD) | |
| response = http_client.context.http_client.get( | |
| "/api/v1/asset/bytes/blob", params={"id": asset["id"]} | |
| ) | |
| assert response.status_code == 200 | |
| assert response.content == PAYLOAD | |
| def test_multi_asset_roundtrip(http_client, tmp_path): | |
| """A bytes node backed by multiple assets reports each size and is downloadable | |
| in `num` order; concatenating the parts reconstructs the original payload.""" | |
| chunks = [PAYLOAD[i : i + 7] for i in range(0, len(PAYLOAD), 7)] # noqa: E203 | |
| # Registered out of order; `num` defines the canonical order. | |
| _register_multi_asset_node(http_client, tmp_path, chunks, key="multi", reverse=True) | |
| meta = http_client.context.http_client.get( | |
| "/api/v1/metadata/multi", params={"include_data_sources": True} | |
| ) | |
| ds = meta.json()["data"]["attributes"]["data_sources"][0] | |
| sorted_assets = sorted(ds["assets"], key=lambda a: a["num"]) | |
| assert [a["size"] for a in sorted_assets] == [len(c) for c in chunks] | |
| parts = [] | |
| for asset in sorted_assets: | |
| r = http_client.context.http_client.get( | |
| "/api/v1/asset/bytes/multi", params={"id": asset["id"]} | |
| ) | |
| assert r.status_code == 200 | |
| parts.append(r.content) | |
| assert b"".join(parts) == PAYLOAD | |
| # __repr__ on a multi-asset node sums sizes and pluralizes "assets". | |
| assert ( | |
| repr(http_client["multi"].include_data_sources()) | |
| == f"<BytesClient {len(PAYLOAD)} B across {len(chunks)} assets>" | |
| ) | |
| def test_asset_bytes_gated_by_expose_raw_assets(tmpdir, tmp_path): | |
| """With `expose_raw_assets=False`, /asset/bytes returns 403 even though | |
| metadata access (and therefore asset-id discovery) remains unrestricted.""" | |
| catalog = in_memory(writable_storage=str(tmpdir)) | |
| app = build_app(catalog, server_settings={"expose_raw_assets": False}) | |
| with Context.from_app(app) as ctx: | |
| client = from_context(ctx) | |
| _register_bytes_node(client, tmp_path, PAYLOAD, key="blob") | |
| meta = client.context.http_client.get( | |
| "/api/v1/metadata/blob", params={"include_data_sources": True} | |
| ) | |
| asset_id = meta.json()["data"]["attributes"]["data_sources"][0]["assets"][0][ | |
| "id" | |
| ] | |
| response = client.context.http_client.get( | |
| "/api/v1/asset/bytes/blob", params={"id": asset_id} | |
| ) | |
| assert response.status_code == 403 | |
| def test_registers_with_arbitrary_mimetype(tmpdir, tmp_path): | |
| """Bytes is the fallback adapter for any unknown-structure mimetype; mapping | |
| `application/pdf` to BytesAdapter is enough to register PDFs as bytes nodes.""" | |
| catalog = in_memory( | |
| writable_storage=str(tmpdir), | |
| adapters_by_mimetype={"application/pdf": BytesAdapter}, | |
| ) | |
| with Context.from_app(build_app(catalog)) as ctx: | |
| client = from_context(ctx) | |
| _register_bytes_node( | |
| client, tmp_path, b"%PDF-1.4 fake", mimetype="application/pdf", key="doc" | |
| ) | |
| meta = client.context.http_client.get( | |
| "/api/v1/metadata/doc", params={"include_data_sources": True} | |
| ) | |
| ds = meta.json()["data"]["attributes"]["data_sources"][0] | |
| assert ds["mimetype"] == "application/pdf" | |
| assert ds["structure_family"] == "bytes" | |
| def test_raw_export_single_asset(http_client, tmp_path, destination): | |
| """`BaseClient.raw_export()` writes a single-asset bytes payload to one handle. | |
| Bytes nodes return the generic `BaseClient` (no family-specific client), so | |
| `raw_export` is the high-level user-facing download API. The destination may | |
| be either a directory on disk or an in-memory `MutableMapping`; in both | |
| cases a single-asset node produces exactly one handle keyed by the | |
| server-supplied filename. | |
| """ | |
| _register_bytes_node(http_client, tmp_path, PAYLOAD, key="blob") | |
| handles = http_client["blob"].raw_export(destination.arg) | |
| assert len(handles) == 1 | |
| assert Path(handles[0]).name == "blob.bin" | |
| assert destination.read(handles[0]) == PAYLOAD | |
| def test_raw_export_multi_asset(http_client, tmp_path, destination): | |
| """`raw_export()` on a multi-asset bytes node namespaces handles by asset id. | |
| On disk each asset lands in `<dest>/<asset_id>/<filename>`; in a mapping the | |
| keys take the form `<asset_id>/<filename>`. In either layout, ordering the | |
| handles by the asset's server-side `num` reproduces the original payload. | |
| """ | |
| chunks = [PAYLOAD[i : i + 7] for i in range(0, len(PAYLOAD), 7)] # noqa: E203 | |
| _register_multi_asset_node(http_client, tmp_path, chunks, key="multi") | |
| handles = http_client["multi"].raw_export(destination.arg) | |
| assert len(handles) == len(chunks) | |
| ds = http_client.context.http_client.get( | |
| "/api/v1/metadata/multi", params={"include_data_sources": True} | |
| ).json()["data"]["attributes"]["data_sources"][0] | |
| num_by_id = {a["id"]: a["num"] for a in ds["assets"]} | |
| ordered = sorted(handles, key=lambda h: num_by_id[destination.asset_id(h)]) | |
| assert b"".join(destination.read(h) for h in ordered) == PAYLOAD | |
| def test_put_asset_backfills_size_on_reregistration(http_client, tmp_path): | |
| """Re-registering an asset whose stored `size` is stale (or NULL) updates | |
| it. `_put_asset` matches existing assets by `data_uri`; when the new | |
| record carries a non-null `size` that differs from the stored value, | |
| the row is updated. This lets a re-`register` repair pre-existing rows | |
| that predate `Asset.size` being populated by the client.""" | |
| # First registration: register with size=None to simulate a legacy row. | |
| p = tmp_path / "blob.bin" | |
| p.write_bytes(PAYLOAD) | |
| http_client.new( | |
| structure_family=StructureFamily.bytes, | |
| data_sources=[ | |
| DataSource( | |
| mimetype="application/octet-stream", | |
| assets=[ | |
| Asset( | |
| data_uri=p.as_uri(), | |
| is_directory=False, | |
| size=None, | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ], | |
| structure_family=StructureFamily.bytes, | |
| structure=BytesStructure(), | |
| management=Management.external, | |
| ) | |
| ], | |
| key="legacy", | |
| ) | |
| meta = http_client.context.http_client.get( | |
| "/api/v1/metadata/legacy", params={"include_data_sources": True} | |
| ).json() | |
| assert meta["data"]["attributes"]["data_sources"][0]["assets"][0]["size"] is None | |
| # Re-register a new node referencing the same data_uri, this time with | |
| # size populated. The shared underlying asset row should be updated. | |
| http_client.new( | |
| structure_family=StructureFamily.bytes, | |
| data_sources=[ | |
| DataSource( | |
| mimetype="application/octet-stream", | |
| assets=[ | |
| Asset( | |
| data_uri=p.as_uri(), | |
| is_directory=False, | |
| size=len(PAYLOAD), | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ], | |
| structure_family=StructureFamily.bytes, | |
| structure=BytesStructure(), | |
| management=Management.external, | |
| ) | |
| ], | |
| key="refreshed", | |
| ) | |
| for key in ("legacy", "refreshed"): | |
| meta = http_client.context.http_client.get( | |
| f"/api/v1/metadata/{key}", params={"include_data_sources": True} | |
| ).json() | |
| asset = meta["data"]["attributes"]["data_sources"][0]["assets"][0] | |
| assert asset["size"] == len(PAYLOAD) | |
| def test_raw_export_destination_directory_kwarg_is_deprecated(http_client, tmp_path): | |
| """The old `destination_directory` keyword still works but emits | |
| `DeprecationWarning`. Passing both the new and old names raises.""" | |
| _register_bytes_node(http_client, tmp_path, PAYLOAD, key="blob") | |
| dest = tmp_path / "out" | |
| dest.mkdir() | |
| with warnings.catch_warnings(record=True) as caught: | |
| warnings.simplefilter("always") | |
| paths = http_client["blob"].raw_export(destination_directory=dest) | |
| assert len(paths) == 1 | |
| assert Path(paths[0]).read_bytes() == PAYLOAD | |
| assert any( | |
| issubclass(w.category, DeprecationWarning) | |
| and "destination_directory" in str(w.message) | |
| for w in caught | |
| ) | |
| with pytest.raises(TypeError, match="both 'destination'"): | |
| http_client["blob"].raw_export(dest, destination_directory=dest) | |
| def test_raw_export_blosc2_streaming_not_truncated(http_client, tmp_path): | |
| """A streaming `blosc2`-encoded body must round-trip in full. | |
| `/asset/bytes` streams via `FileResponse` in 64 KiB chunks, and | |
| `BloscBuffer` compresses each chunk into an independent blosc2 frame, so the | |
| wire body is `frame0 ++ frame1 ++ ...`. | |
| """ | |
| # Comfortably larger than the 65536-byte FileResponse chunk size, so the | |
| # body spans several chunks (i.e. several blosc2 frames). Compressible so | |
| # the middleware actually engages. | |
| payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB, > 2 chunks | |
| assert len(payload) > 2 * 65536 | |
| _register_bytes_node(http_client, tmp_path, payload, key="big") | |
| # Prefer blosc2 so the server selects it (server-preferred among these) and | |
| # the multi-frame streaming path is exercised end-to-end. | |
| http_client.context.http_client.headers["Accept-Encoding"] = "blosc2, zstd, gzip" | |
| dest = tmp_path / "out" | |
| dest.mkdir() | |
| paths = http_client["big"].raw_export(dest) | |
| assert len(paths) == 1 | |
| downloaded = Path(paths[0]).read_bytes() | |
| # The bug truncated this to exactly 65536 bytes; assert full length + bytes. | |
| assert len(downloaded) == len(payload) | |
| assert downloaded == payload | |
| # Confirm we actually exercised the blosc2 streaming path (and its Content- | |
| # Encoding), not a fallback encoding. | |
| ds = http_client.context.http_client.get( | |
| "/api/v1/metadata/big", params={"include_data_sources": True} | |
| ).json()["data"]["attributes"]["data_sources"][0] | |
| asset_id = ds["assets"][0]["id"] | |
| response = http_client.context.http_client.get( | |
| "/api/v1/asset/bytes/big", params={"id": asset_id} | |
| ) | |
| assert response.status_code == 200 | |
| assert response.headers.get("Content-Encoding") == "blosc2" | |
| assert response.content == payload | |
| def _asset_bytes_exchange(history): | |
| """Return the (request, response) pair for the `/asset/bytes` download.""" | |
| for request, response in zip(history.requests, history.responses): | |
| if "/asset/bytes/" in str(request.url): | |
| return request, response | |
| raise AssertionError("no /asset/bytes request was recorded") | |
| def test_raw_export_streams_with_compression_but_not_blosc2(http_client, tmp_path): | |
| """By default `raw_export` keeps on-the-fly compression but excludes blosc2. | |
| blosc2's client decoder buffers the whole body (no streaming), so raw-asset | |
| downloads advertise every supported encoding *except* blosc2. The server | |
| then streams via zstd, which decompresses incrementally. | |
| """ | |
| payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB, compressible | |
| _register_bytes_node(http_client, tmp_path, payload, key="big") | |
| dest = tmp_path / "out" | |
| dest.mkdir() | |
| with record_history() as history: | |
| paths = http_client["big"].raw_export(dest) | |
| assert Path(paths[0]).read_bytes() == payload | |
| request, response = _asset_bytes_exchange(history) | |
| # blosc2 must not be advertised on the download request... | |
| assert request.headers["Accept-Encoding"] == STREAMING_ACCEPT_ENCODING | |
| assert "blosc2" not in request.headers["Accept-Encoding"] | |
| # ...and the server streams a compressed (zstd) response. | |
| assert response.headers.get("Content-Encoding") == "zstd" | |
| def test_raw_export_compression_false_uses_identity(http_client, tmp_path): | |
| """`compression=False` downloads uncompressed: `Accept-Encoding: identity`, | |
| no `Content-Encoding`, and a `Content-Length` (so the bar is determinate).""" | |
| payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB, compressible | |
| _register_bytes_node(http_client, tmp_path, payload, key="big") | |
| dest = tmp_path / "out" | |
| dest.mkdir() | |
| with record_history() as history: | |
| paths = http_client["big"].raw_export(dest, compression=False) | |
| assert Path(paths[0]).read_bytes() == payload | |
| request, response = _asset_bytes_exchange(history) | |
| assert request.headers["Accept-Encoding"] == "identity" | |
| assert response.headers.get("Content-Encoding") in (None, "identity") | |
| assert int(response.headers["Content-Length"]) == len(payload) | |
| def _captured_task_totals(client, key, **export_kwargs): | |
| """Run `raw_export` and record the `total` each progress task is seeded | |
| with (via `Progress.add_task`).""" | |
| totals = [] | |
| original = Progress.add_task | |
| def spy(self, description, *args, total=None, **kwargs): | |
| totals.append(total) | |
| return original(self, description, *args, total=total, **kwargs) | |
| with mock.patch.object(Progress, "add_task", spy): | |
| client[key].raw_export(**export_kwargs) | |
| return totals | |
| def test_raw_export_seeds_progress_total_from_asset_size(http_client, tmp_path): | |
| """The progress task is seeded with the known asset size, so the bar shows | |
| the right total even when the server omits `Content-Length` (compressed | |
| streaming). When the size is unknown the total is `None` (indeterminate).""" | |
| payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB | |
| _register_bytes_node(http_client, tmp_path, payload, key="known") | |
| _register_bytes_node(http_client, tmp_path, payload, key="unknown", size=None) | |
| known_dest = tmp_path / "known_out" | |
| known_dest.mkdir() | |
| assert _captured_task_totals(http_client, "known", destination=known_dest) == [ | |
| len(payload) | |
| ] | |
| unknown_dest = tmp_path / "unknown_out" | |
| unknown_dest.mkdir() | |
| assert _captured_task_totals(http_client, "unknown", destination=unknown_dest) == [ | |
| None | |
| ] | |
| def test_raw_export_unknown_size_roundtrips(http_client, tmp_path, compression): | |
| """A `bytes` node whose asset size is unknown (e.g. a legacy row) exports | |
| correctly under both compression modes. With compression the bar is | |
| indeterminate (no `Content-Length`); without it, `Content-Length` is present | |
| so the bar is determinate.""" | |
| payload = os.urandom(200 * 1024) # incompressible so zstd does not shrink it | |
| _register_bytes_node(http_client, tmp_path, payload, key="blob", size=None) | |
| dest = tmp_path / f"out_{compression}" | |
| dest.mkdir() | |
| with record_history() as history: | |
| paths = http_client["blob"].raw_export(dest, compression=compression) | |
| assert Path(paths[0]).read_bytes() == payload | |
| _, response = _asset_bytes_exchange(history) | |
| if compression: | |
| assert "Content-Length" not in response.headers | |
| else: | |
| assert int(response.headers["Content-Length"]) == len(payload) | |
| def test_download_rejects_mismatched_totals(http_client): | |
| """`download()` requires `totals` (when given) to be parallel to `urls`.""" | |
| with pytest.raises(ValueError, match="as many totals as URLs"): | |
| download( | |
| http_client.context.http_client, | |
| ["u1", "u2"], | |
| ["t1", "t2"], | |
| totals=[1], | |
| ) | |
| def test_raw_export_handles_missing_content_length(http_client, tmp_path): | |
| """Regression: large compressed responses have no `Content-Length` header | |
| (the compression middleware strips it when streaming a chunked body). | |
| `raw_export` must not crash with `KeyError('Content-Length')` and should | |
| show an indeterminate progress bar instead.""" | |
| # A payload comfortably larger than the compression middleware's | |
| # minimum_size (500 bytes) so the server will actually compress and stream. | |
| large_payload = (b"the quick brown fox " * 1024) * 64 # ~1.3 MiB | |
| p = tmp_path / "big.bin" | |
| p.write_bytes(large_payload) | |
| http_client.new( | |
| structure_family=StructureFamily.bytes, | |
| data_sources=[ | |
| DataSource( | |
| mimetype="application/octet-stream", | |
| assets=[ | |
| Asset( | |
| data_uri=p.as_uri(), | |
| is_directory=False, | |
| size=len(large_payload), | |
| parameter="data_uris", | |
| num=0, | |
| ) | |
| ], | |
| structure_family=StructureFamily.bytes, | |
| structure=BytesStructure(), | |
| management=Management.external, | |
| ) | |
| ], | |
| key="big", | |
| ) | |
| # Force the server's compression middleware to engage and strip | |
| # Content-Length on the streaming branch. | |
| http_client.context.http_client.headers["Accept-Encoding"] = "zstd" | |
| dest = tmp_path / "out" | |
| dest.mkdir() | |
| paths = http_client["big"].raw_export(dest) | |
| assert len(paths) == 1 | |
| assert Path(paths[0]).read_bytes() == large_payload | |
| # Exercise the non-byte units in __repr__'s size formatting. | |
| expected_size = f"{len(large_payload) / 1024 / 1024:.1f} MB" | |
| assert ( | |
| repr(http_client["big"].include_data_sources()) | |
| == f"<BytesClient {expected_size} across 1 asset>" | |
| ) | |
Xet Storage Details
- Size:
- 23.2 kB
- Xet hash:
- 196f3ca61af06cec74fc4012eb4c2317913eacfdf56f4092dfadb469a147a9e8
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.