Buckets:

download
raw
23.2 kB
"""Integration tests for the `bytes` structure family.
The `bytes` family has no dedicated content endpoint and no family-specific
client. Clients register an opaque payload, then download each underlying
asset through `/asset/bytes/{path}?id=N`, gated by `settings.expose_raw_assets`.
These tests exercise that full round-trip end-to-end.
"""
import os
import warnings
from pathlib import Path
from unittest import mock
import pytest
from rich.progress import Progress
from tiled.adapters.bytes import BytesAdapter
from tiled.catalog import in_memory
from tiled.client import Context, from_context, record_history
from tiled.client.bytes import BytesClient
from tiled.client.download import STREAMING_ACCEPT_ENCODING, download
from tiled.server.app import build_app
from tiled.structures.bytes import BytesStructure
from tiled.structures.core import StructureFamily
from tiled.structures.data_source import Asset, DataSource, Management
PAYLOAD = b"the quick brown fox jumps over the lazy dog"
@pytest.fixture
def http_client(tmpdir):
catalog = in_memory(writable_storage=str(tmpdir))
with Context.from_app(build_app(catalog)) as ctx:
yield from_context(ctx)
def _register_bytes_node(
client,
tmp_path,
payload,
key="blob",
mimetype="application/octet-stream",
size="auto",
):
"""Write `payload` to a file and register it as an external bytes node.
`size` defaults to the payload length; pass `None` to register an asset
whose size is unknown (mimicking a legacy row)."""
p = tmp_path / f"{key}.bin"
p.write_bytes(payload)
data_source = DataSource(
mimetype=mimetype,
assets=[
Asset(
data_uri=p.as_uri(),
is_directory=False,
size=len(payload) if size == "auto" else size,
parameter="data_uris",
num=0,
)
],
structure_family=StructureFamily.bytes,
structure=BytesStructure(),
management=Management.external,
)
return client.new(
structure_family=StructureFamily.bytes,
data_sources=[data_source],
key=key,
)
def _register_multi_asset_node(client, tmp_path, chunks, key="multi", reverse=False):
"""Write each chunk to its own file and register them as one bytes node.
`num` is set to the chunk's index, so the original payload can be recovered
by ordering downloaded parts by `num`. When `reverse=True`, the assets are
submitted in the *opposite* order to verify the server canonicalises by `num`.
"""
assets = []
for i, chunk in enumerate(chunks):
p = tmp_path / f"c{i:02d}.bin"
p.write_bytes(chunk)
assets.append(
Asset(
data_uri=p.as_uri(),
is_directory=False,
size=len(chunk),
parameter="data_uris",
num=i,
)
)
client.new(
structure_family=StructureFamily.bytes,
data_sources=[
DataSource(
mimetype="application/octet-stream",
assets=list(reversed(assets)) if reverse else assets,
structure_family=StructureFamily.bytes,
structure=BytesStructure(),
management=Management.external,
)
],
key=key,
)
class _DiskDestination:
"""`raw_export` target backed by a directory on disk."""
def __init__(self, tmp_path):
self.dest = tmp_path / "out"
self.dest.mkdir()
@property
def arg(self):
return self.dest
def read(self, handle):
return Path(handle).read_bytes()
def asset_id(self, handle):
# Multi-asset disk layout: `<dest>/<asset_id>/<filename>`.
return int(Path(handle).parent.name)
class _MappingDestination:
"""`raw_export` target backed by an in-memory `dict` of `BytesIO` buffers."""
def __init__(self, tmp_path):
self.buffers = {}
@property
def arg(self):
return self.buffers
def read(self, handle):
return self.buffers[handle].read()
def asset_id(self, handle):
# Multi-asset mapping layout: `<asset_id>/<filename>`.
return int(handle.split("/")[0])
@pytest.fixture(params=["disk", "mapping"])
def destination(request, tmp_path):
return {"disk": _DiskDestination, "mapping": _MappingDestination}[request.param](
tmp_path
)
def test_roundtrip_register_and_download(http_client, tmp_path):
"""Register a bytes node, inspect its metadata, and download the asset.
Covers the full path: structure_family wiring, default mimetype,
Asset.size population, asset-id discovery, and content round-trip via /asset/bytes.
Also exercises BytesClient dispatch and its size-aware __repr__.
"""
handle = _register_bytes_node(http_client, tmp_path, PAYLOAD, key="blob")
assert handle.item["attributes"]["structure_family"] == "bytes"
# Dispatch resolves bytes nodes to BytesClient, not generic BaseClient.
assert isinstance(http_client["blob"], BytesClient)
# Without include_data_sources(), __repr__ falls back to the class name
# and must not trigger a network round-trip.
assert repr(http_client["blob"]) == "<BytesClient>"
# With data sources cached, __repr__ reports total size and asset count.
assert (
repr(http_client["blob"].include_data_sources())
== f"<BytesClient {len(PAYLOAD)} B across 1 asset>"
)
meta = http_client.context.http_client.get(
"/api/v1/metadata/blob", params={"include_data_sources": True}
)
ds = meta.json()["data"]["attributes"]["data_sources"][0]
assert ds["mimetype"] == "application/octet-stream"
asset = ds["assets"][0]
assert asset["size"] == len(PAYLOAD)
response = http_client.context.http_client.get(
"/api/v1/asset/bytes/blob", params={"id": asset["id"]}
)
assert response.status_code == 200
assert response.content == PAYLOAD
def test_multi_asset_roundtrip(http_client, tmp_path):
"""A bytes node backed by multiple assets reports each size and is downloadable
in `num` order; concatenating the parts reconstructs the original payload."""
chunks = [PAYLOAD[i : i + 7] for i in range(0, len(PAYLOAD), 7)] # noqa: E203
# Registered out of order; `num` defines the canonical order.
_register_multi_asset_node(http_client, tmp_path, chunks, key="multi", reverse=True)
meta = http_client.context.http_client.get(
"/api/v1/metadata/multi", params={"include_data_sources": True}
)
ds = meta.json()["data"]["attributes"]["data_sources"][0]
sorted_assets = sorted(ds["assets"], key=lambda a: a["num"])
assert [a["size"] for a in sorted_assets] == [len(c) for c in chunks]
parts = []
for asset in sorted_assets:
r = http_client.context.http_client.get(
"/api/v1/asset/bytes/multi", params={"id": asset["id"]}
)
assert r.status_code == 200
parts.append(r.content)
assert b"".join(parts) == PAYLOAD
# __repr__ on a multi-asset node sums sizes and pluralizes "assets".
assert (
repr(http_client["multi"].include_data_sources())
== f"<BytesClient {len(PAYLOAD)} B across {len(chunks)} assets>"
)
def test_asset_bytes_gated_by_expose_raw_assets(tmpdir, tmp_path):
"""With `expose_raw_assets=False`, /asset/bytes returns 403 even though
metadata access (and therefore asset-id discovery) remains unrestricted."""
catalog = in_memory(writable_storage=str(tmpdir))
app = build_app(catalog, server_settings={"expose_raw_assets": False})
with Context.from_app(app) as ctx:
client = from_context(ctx)
_register_bytes_node(client, tmp_path, PAYLOAD, key="blob")
meta = client.context.http_client.get(
"/api/v1/metadata/blob", params={"include_data_sources": True}
)
asset_id = meta.json()["data"]["attributes"]["data_sources"][0]["assets"][0][
"id"
]
response = client.context.http_client.get(
"/api/v1/asset/bytes/blob", params={"id": asset_id}
)
assert response.status_code == 403
def test_registers_with_arbitrary_mimetype(tmpdir, tmp_path):
"""Bytes is the fallback adapter for any unknown-structure mimetype; mapping
`application/pdf` to BytesAdapter is enough to register PDFs as bytes nodes."""
catalog = in_memory(
writable_storage=str(tmpdir),
adapters_by_mimetype={"application/pdf": BytesAdapter},
)
with Context.from_app(build_app(catalog)) as ctx:
client = from_context(ctx)
_register_bytes_node(
client, tmp_path, b"%PDF-1.4 fake", mimetype="application/pdf", key="doc"
)
meta = client.context.http_client.get(
"/api/v1/metadata/doc", params={"include_data_sources": True}
)
ds = meta.json()["data"]["attributes"]["data_sources"][0]
assert ds["mimetype"] == "application/pdf"
assert ds["structure_family"] == "bytes"
def test_raw_export_single_asset(http_client, tmp_path, destination):
"""`BaseClient.raw_export()` writes a single-asset bytes payload to one handle.
Bytes nodes return the generic `BaseClient` (no family-specific client), so
`raw_export` is the high-level user-facing download API. The destination may
be either a directory on disk or an in-memory `MutableMapping`; in both
cases a single-asset node produces exactly one handle keyed by the
server-supplied filename.
"""
_register_bytes_node(http_client, tmp_path, PAYLOAD, key="blob")
handles = http_client["blob"].raw_export(destination.arg)
assert len(handles) == 1
assert Path(handles[0]).name == "blob.bin"
assert destination.read(handles[0]) == PAYLOAD
def test_raw_export_multi_asset(http_client, tmp_path, destination):
"""`raw_export()` on a multi-asset bytes node namespaces handles by asset id.
On disk each asset lands in `<dest>/<asset_id>/<filename>`; in a mapping the
keys take the form `<asset_id>/<filename>`. In either layout, ordering the
handles by the asset's server-side `num` reproduces the original payload.
"""
chunks = [PAYLOAD[i : i + 7] for i in range(0, len(PAYLOAD), 7)] # noqa: E203
_register_multi_asset_node(http_client, tmp_path, chunks, key="multi")
handles = http_client["multi"].raw_export(destination.arg)
assert len(handles) == len(chunks)
ds = http_client.context.http_client.get(
"/api/v1/metadata/multi", params={"include_data_sources": True}
).json()["data"]["attributes"]["data_sources"][0]
num_by_id = {a["id"]: a["num"] for a in ds["assets"]}
ordered = sorted(handles, key=lambda h: num_by_id[destination.asset_id(h)])
assert b"".join(destination.read(h) for h in ordered) == PAYLOAD
def test_put_asset_backfills_size_on_reregistration(http_client, tmp_path):
"""Re-registering an asset whose stored `size` is stale (or NULL) updates
it. `_put_asset` matches existing assets by `data_uri`; when the new
record carries a non-null `size` that differs from the stored value,
the row is updated. This lets a re-`register` repair pre-existing rows
that predate `Asset.size` being populated by the client."""
# First registration: register with size=None to simulate a legacy row.
p = tmp_path / "blob.bin"
p.write_bytes(PAYLOAD)
http_client.new(
structure_family=StructureFamily.bytes,
data_sources=[
DataSource(
mimetype="application/octet-stream",
assets=[
Asset(
data_uri=p.as_uri(),
is_directory=False,
size=None,
parameter="data_uris",
num=0,
)
],
structure_family=StructureFamily.bytes,
structure=BytesStructure(),
management=Management.external,
)
],
key="legacy",
)
meta = http_client.context.http_client.get(
"/api/v1/metadata/legacy", params={"include_data_sources": True}
).json()
assert meta["data"]["attributes"]["data_sources"][0]["assets"][0]["size"] is None
# Re-register a new node referencing the same data_uri, this time with
# size populated. The shared underlying asset row should be updated.
http_client.new(
structure_family=StructureFamily.bytes,
data_sources=[
DataSource(
mimetype="application/octet-stream",
assets=[
Asset(
data_uri=p.as_uri(),
is_directory=False,
size=len(PAYLOAD),
parameter="data_uris",
num=0,
)
],
structure_family=StructureFamily.bytes,
structure=BytesStructure(),
management=Management.external,
)
],
key="refreshed",
)
for key in ("legacy", "refreshed"):
meta = http_client.context.http_client.get(
f"/api/v1/metadata/{key}", params={"include_data_sources": True}
).json()
asset = meta["data"]["attributes"]["data_sources"][0]["assets"][0]
assert asset["size"] == len(PAYLOAD)
def test_raw_export_destination_directory_kwarg_is_deprecated(http_client, tmp_path):
"""The old `destination_directory` keyword still works but emits
`DeprecationWarning`. Passing both the new and old names raises."""
_register_bytes_node(http_client, tmp_path, PAYLOAD, key="blob")
dest = tmp_path / "out"
dest.mkdir()
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
paths = http_client["blob"].raw_export(destination_directory=dest)
assert len(paths) == 1
assert Path(paths[0]).read_bytes() == PAYLOAD
assert any(
issubclass(w.category, DeprecationWarning)
and "destination_directory" in str(w.message)
for w in caught
)
with pytest.raises(TypeError, match="both 'destination'"):
http_client["blob"].raw_export(dest, destination_directory=dest)
def test_raw_export_blosc2_streaming_not_truncated(http_client, tmp_path):
"""A streaming `blosc2`-encoded body must round-trip in full.
`/asset/bytes` streams via `FileResponse` in 64 KiB chunks, and
`BloscBuffer` compresses each chunk into an independent blosc2 frame, so the
wire body is `frame0 ++ frame1 ++ ...`.
"""
# Comfortably larger than the 65536-byte FileResponse chunk size, so the
# body spans several chunks (i.e. several blosc2 frames). Compressible so
# the middleware actually engages.
payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB, > 2 chunks
assert len(payload) > 2 * 65536
_register_bytes_node(http_client, tmp_path, payload, key="big")
# Prefer blosc2 so the server selects it (server-preferred among these) and
# the multi-frame streaming path is exercised end-to-end.
http_client.context.http_client.headers["Accept-Encoding"] = "blosc2, zstd, gzip"
dest = tmp_path / "out"
dest.mkdir()
paths = http_client["big"].raw_export(dest)
assert len(paths) == 1
downloaded = Path(paths[0]).read_bytes()
# The bug truncated this to exactly 65536 bytes; assert full length + bytes.
assert len(downloaded) == len(payload)
assert downloaded == payload
# Confirm we actually exercised the blosc2 streaming path (and its Content-
# Encoding), not a fallback encoding.
ds = http_client.context.http_client.get(
"/api/v1/metadata/big", params={"include_data_sources": True}
).json()["data"]["attributes"]["data_sources"][0]
asset_id = ds["assets"][0]["id"]
response = http_client.context.http_client.get(
"/api/v1/asset/bytes/big", params={"id": asset_id}
)
assert response.status_code == 200
assert response.headers.get("Content-Encoding") == "blosc2"
assert response.content == payload
def _asset_bytes_exchange(history):
"""Return the (request, response) pair for the `/asset/bytes` download."""
for request, response in zip(history.requests, history.responses):
if "/asset/bytes/" in str(request.url):
return request, response
raise AssertionError("no /asset/bytes request was recorded")
def test_raw_export_streams_with_compression_but_not_blosc2(http_client, tmp_path):
"""By default `raw_export` keeps on-the-fly compression but excludes blosc2.
blosc2's client decoder buffers the whole body (no streaming), so raw-asset
downloads advertise every supported encoding *except* blosc2. The server
then streams via zstd, which decompresses incrementally.
"""
payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB, compressible
_register_bytes_node(http_client, tmp_path, payload, key="big")
dest = tmp_path / "out"
dest.mkdir()
with record_history() as history:
paths = http_client["big"].raw_export(dest)
assert Path(paths[0]).read_bytes() == payload
request, response = _asset_bytes_exchange(history)
# blosc2 must not be advertised on the download request...
assert request.headers["Accept-Encoding"] == STREAMING_ACCEPT_ENCODING
assert "blosc2" not in request.headers["Accept-Encoding"]
# ...and the server streams a compressed (zstd) response.
assert response.headers.get("Content-Encoding") == "zstd"
def test_raw_export_compression_false_uses_identity(http_client, tmp_path):
"""`compression=False` downloads uncompressed: `Accept-Encoding: identity`,
no `Content-Encoding`, and a `Content-Length` (so the bar is determinate)."""
payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB, compressible
_register_bytes_node(http_client, tmp_path, payload, key="big")
dest = tmp_path / "out"
dest.mkdir()
with record_history() as history:
paths = http_client["big"].raw_export(dest, compression=False)
assert Path(paths[0]).read_bytes() == payload
request, response = _asset_bytes_exchange(history)
assert request.headers["Accept-Encoding"] == "identity"
assert response.headers.get("Content-Encoding") in (None, "identity")
assert int(response.headers["Content-Length"]) == len(payload)
def _captured_task_totals(client, key, **export_kwargs):
"""Run `raw_export` and record the `total` each progress task is seeded
with (via `Progress.add_task`)."""
totals = []
original = Progress.add_task
def spy(self, description, *args, total=None, **kwargs):
totals.append(total)
return original(self, description, *args, total=total, **kwargs)
with mock.patch.object(Progress, "add_task", spy):
client[key].raw_export(**export_kwargs)
return totals
def test_raw_export_seeds_progress_total_from_asset_size(http_client, tmp_path):
"""The progress task is seeded with the known asset size, so the bar shows
the right total even when the server omits `Content-Length` (compressed
streaming). When the size is unknown the total is `None` (indeterminate)."""
payload = (b"the quick brown fox " * 1024) * 8 # ~160 KiB
_register_bytes_node(http_client, tmp_path, payload, key="known")
_register_bytes_node(http_client, tmp_path, payload, key="unknown", size=None)
known_dest = tmp_path / "known_out"
known_dest.mkdir()
assert _captured_task_totals(http_client, "known", destination=known_dest) == [
len(payload)
]
unknown_dest = tmp_path / "unknown_out"
unknown_dest.mkdir()
assert _captured_task_totals(http_client, "unknown", destination=unknown_dest) == [
None
]
@pytest.mark.parametrize("compression", [True, False])
def test_raw_export_unknown_size_roundtrips(http_client, tmp_path, compression):
"""A `bytes` node whose asset size is unknown (e.g. a legacy row) exports
correctly under both compression modes. With compression the bar is
indeterminate (no `Content-Length`); without it, `Content-Length` is present
so the bar is determinate."""
payload = os.urandom(200 * 1024) # incompressible so zstd does not shrink it
_register_bytes_node(http_client, tmp_path, payload, key="blob", size=None)
dest = tmp_path / f"out_{compression}"
dest.mkdir()
with record_history() as history:
paths = http_client["blob"].raw_export(dest, compression=compression)
assert Path(paths[0]).read_bytes() == payload
_, response = _asset_bytes_exchange(history)
if compression:
assert "Content-Length" not in response.headers
else:
assert int(response.headers["Content-Length"]) == len(payload)
def test_download_rejects_mismatched_totals(http_client):
"""`download()` requires `totals` (when given) to be parallel to `urls`."""
with pytest.raises(ValueError, match="as many totals as URLs"):
download(
http_client.context.http_client,
["u1", "u2"],
["t1", "t2"],
totals=[1],
)
def test_raw_export_handles_missing_content_length(http_client, tmp_path):
"""Regression: large compressed responses have no `Content-Length` header
(the compression middleware strips it when streaming a chunked body).
`raw_export` must not crash with `KeyError('Content-Length')` and should
show an indeterminate progress bar instead."""
# A payload comfortably larger than the compression middleware's
# minimum_size (500 bytes) so the server will actually compress and stream.
large_payload = (b"the quick brown fox " * 1024) * 64 # ~1.3 MiB
p = tmp_path / "big.bin"
p.write_bytes(large_payload)
http_client.new(
structure_family=StructureFamily.bytes,
data_sources=[
DataSource(
mimetype="application/octet-stream",
assets=[
Asset(
data_uri=p.as_uri(),
is_directory=False,
size=len(large_payload),
parameter="data_uris",
num=0,
)
],
structure_family=StructureFamily.bytes,
structure=BytesStructure(),
management=Management.external,
)
],
key="big",
)
# Force the server's compression middleware to engage and strip
# Content-Length on the streaming branch.
http_client.context.http_client.headers["Accept-Encoding"] = "zstd"
dest = tmp_path / "out"
dest.mkdir()
paths = http_client["big"].raw_export(dest)
assert len(paths) == 1
assert Path(paths[0]).read_bytes() == large_payload
# Exercise the non-byte units in __repr__'s size formatting.
expected_size = f"{len(large_payload) / 1024 / 1024:.1f} MB"
assert (
repr(http_client["big"].include_data_sources())
== f"<BytesClient {expected_size} across 1 asset>"
)

Xet Storage Details

Size:
23.2 kB
·
Xet hash:
196f3ca61af06cec74fc4012eb4c2317913eacfdf56f4092dfadb469a147a9e8

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.