Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /bluesky_tiled /tests /test_websockets.py
import contextlib
import datetime
import sys
import urllib.parse
import dask.array
import jose.jwt
import msgpack
import numpy as np
import pandas as pd
import pyarrow as pa
import pytest
from starlette.testclient import WebSocketDenialResponse
from tiled.client import from_context
from tiled.config import parse_configs
pytestmark = pytest.mark.skipif(
sys.platform == "win32", reason="Requires Redis service"
)
def send_ws_updates(client, update_func, start=1, count=1, persist=None):
"""Helper to send updates via Tiled client in websocket tests."""
for i in range(start, start + count):
new_arr = np.arange(10) + i
update_func(client, new_arr, i, persist=persist)
# An update_func for send_ws_updates
def overwrite_array(client, new_arr, seq_num, persist=None):
_ = seq_num # seq_num is unused for these updates
client.write(new_arr, persist=persist)
# An update_func for send_ws_updates
def write_array_block(client, new_arr, seq_num, persist=None):
client.write_block(new_arr, block=(seq_num - 1, 0), persist=persist)
# An update_func for send_ws_updates
def patch_array(client, new_arr, seq_num, persist=None):
_ = seq_num # seq_num is unused for these updates
client.patch(new_arr, offset=(0,), persist=persist)
# An update_func for send_ws_updates
def append_array(client, new_arr, seq_num, persist=None):
client.patch(new_arr, offset=(10 * seq_num,), extend=True, persist=persist)
def receive_ws_updates(websocket, count=1):
"""Helper to receive updates in websocket tests."""
# Receive all updates
envelope_format = urllib.parse.parse_qs(websocket.scope["query_string"].decode())[
"envelope_format"
][0]
received = []
for _ in range(count + 1): # +1 for schema
if envelope_format == "json":
msg = websocket.receive_json()
else: # default to msgpack
msg_bytes = websocket.receive_bytes()
msg = msgpack.unpackb(msg_bytes)
received.append(msg)
# Verify all messages received (schema + n updates)
assert len(received) == count + 1
return received
def verify_ws_updates(received, start=1, chunked=False):
"""Verify that we received messages with the expected data"""
for i, msg in enumerate(received):
if i == 0: # schema
assert "type" in msg
assert "version" in msg
else:
assert "type" in msg
assert "timestamp" in msg
assert "payload" in msg
if chunked:
assert msg["shape"] == [1, 10]
else:
assert msg["shape"] == [10]
# Verify payload contains the expected array data
if isinstance(msg["payload"], bytes):
payload_array = np.frombuffer(msg["payload"], dtype=np.int64)
else:
payload_array = np.array(msg["payload"], dtype=np.int64)
expected_array = (np.arange(10) + (start - 1) + i).reshape(msg["shape"])
np.testing.assert_array_equal(
payload_array.reshape(msg["shape"]), expected_array
)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_subscribe_immediately_after_creation_websockets(
tiled_websocket_context, envelope_format
):
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node using Tiled client
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_stream_immediate")
# Connect WebSocket using TestClient with specified envelope format and authorization
with test_client.websocket_connect(
f"/api/v1/stream/single/test_stream_immediate?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Send 3 updates using Tiled client that overwrite the array
send_ws_updates(streaming_node, overwrite_array, count=3)
# Receive and validate all updates
received = receive_ws_updates(websocket, count=3)
verify_ws_updates(received)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_connection_to_non_existent_node(tiled_websocket_context, envelope_format):
"""Test websocket connection to non-existent node returns 404."""
context = tiled_websocket_context
test_client = context.http_client
non_existent_node_id = "definitely_non_existent_websocket_node_99999999"
# Try to connect to websocket for non-existent node
# This should result in an HTTP 404 response during the handshake
response = test_client.get(
f"/api/v1/stream/single/{non_existent_node_id}?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
)
assert response.status_code == 404
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_subscribe_after_first_update_websockets(
tiled_websocket_context, envelope_format
):
"""Client that subscribes after first update sees only subsequent updates."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node using Tiled client
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_stream_after_update")
# Write first update before subscribing
first_update = np.arange(10) + 1
streaming_node.write(first_update)
# Connect WebSocket after first update
with test_client.websocket_connect(
f"/api/v1/stream/single/test_stream_after_update?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Send 2 more updates that overwrite the array
send_ws_updates(streaming_node, overwrite_array, start=2, count=2)
# Should only receive the 2 new updates
received = receive_ws_updates(websocket, count=2)
# Content starts with update #2
verify_ws_updates(received, start=2)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_subscribe_after_first_update_from_beginning_websockets(
tiled_websocket_context, envelope_format
):
"""Client that subscribes after first update but requests from seq_num=0 sees all updates.
Note: seq_num starts at 1 for the first data point. seq_num=0 means "start as far back
as you have" (similar to Bluesky social)
"""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node using Tiled client
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_stream_from_beginning")
# Write first update before subscribing
first_update = np.arange(10) + 1
streaming_node.write(first_update)
# Connect WebSocket requesting from beginning
with test_client.websocket_connect(
f"/api/v1/stream/single/test_stream_from_beginning?envelope_format={envelope_format}&start=0",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Schema
if envelope_format == "json":
schema_msg = websocket.receive_json()
else:
schema_msg_bytes = websocket.receive_bytes()
schema_msg = msgpack.unpackb(schema_msg_bytes)
assert "type" in schema_msg
assert "version" in schema_msg
# First, should receive the initial array creation
if envelope_format == "json":
historical_msg = websocket.receive_json()
else:
historical_msg_bytes = websocket.receive_bytes()
historical_msg = msgpack.unpackb(historical_msg_bytes)
assert "timestamp" in historical_msg
assert "payload" in historical_msg
assert historical_msg["shape"] == [10]
# Verify historical payload (initial array creation - sequence 0)
if envelope_format == "json":
historical_payload = np.array(historical_msg["payload"], dtype=np.int64)
else:
historical_payload = np.frombuffer(
historical_msg["payload"], dtype=np.int64
)
expected_historical = np.arange(10) # Initial array
np.testing.assert_array_equal(historical_payload, expected_historical)
# Next, should receive the first update (sequence 1)
if envelope_format == "json":
first_update_msg = websocket.receive_json()
first_update_payload = np.array(first_update_msg["payload"], dtype=np.int64)
else:
first_update_bytes = websocket.receive_bytes()
first_update_msg = msgpack.unpackb(first_update_bytes)
first_update_payload = np.frombuffer(
first_update_msg["payload"], dtype=np.int64
)
expected_first_update = np.arange(10) + 1
np.testing.assert_array_equal(first_update_payload, expected_first_update)
# Write more updates
for i in range(2, 4):
new_arr = np.arange(10) + i
streaming_node.write(new_arr)
# Receive the new updates
for i in range(2, 4):
if envelope_format == "json":
msg = websocket.receive_json()
else:
msg_bytes = websocket.receive_bytes()
msg = msgpack.unpackb(msg_bytes)
assert "timestamp" in msg
assert "payload" in msg
assert msg["shape"] == [10]
# Verify payload contains the expected array data
if envelope_format == "json":
payload_array = np.array(msg["payload"], dtype=np.int64)
else:
payload_array = np.frombuffer(msg["payload"], dtype=np.int64)
expected_array = np.arange(10) + i
np.testing.assert_array_equal(payload_array, expected_array)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
@pytest.mark.parametrize("write_op", (overwrite_array, patch_array))
@pytest.mark.parametrize("persist", (None, True, False))
def test_updates_persist_write(
tiled_websocket_context, envelope_format, write_op, persist
):
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node using Tiled client
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_stream_immediate")
# Connect WebSocket using TestClient with msgpack format and authorization
with test_client.websocket_connect(
f"/api/v1/stream/single/test_stream_immediate?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Send 3 updates using Tiled client that write values into the array
send_ws_updates(streaming_node, write_op, count=3, persist=persist)
# Receive and validate all updates
received = receive_ws_updates(websocket, count=3)
verify_ws_updates(received)
# Verify values of persisted data
if persist or persist is None:
expected_persisted = np.arange(10) + 3 # Final sent values
else:
expected_persisted = arr # Original values
persisted_data = streaming_node.read()
np.testing.assert_array_equal(persisted_data, expected_persisted)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
@pytest.mark.parametrize("persist", (None, True, False))
def test_updates_persist_write_block(tiled_websocket_context, envelope_format, persist):
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create a streaming chunked array node using Tiled client
_arr = np.array([np.arange(10) for _ in range(3)])
arr = dask.array.from_array(_arr, chunks=(1, 10)) # Chunk along first axis
streaming_node = client.write_array(arr, key="test_stream_immediate")
# Connect WebSocket using TestClient with msgpack format and authorization
with test_client.websocket_connect(
f"/api/v1/stream/single/test_stream_immediate?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Send 3 updates using Tiled client that write values into the array
send_ws_updates(streaming_node, write_array_block, count=3, persist=persist)
# Receive and validate all updates
received = receive_ws_updates(websocket, count=3)
verify_ws_updates(received, chunked=True)
# Verify values of persisted data
if persist or persist is None:
# Combined effect of all sent values
expected_persisted = np.array([np.arange(10) + i for i in range(1, 4)])
else:
# Original values
expected_persisted = arr
persisted_data = streaming_node.read()
np.testing.assert_array_equal(persisted_data, expected_persisted)
# Extending an array with persist=False is not yet supported
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
@pytest.mark.parametrize("persist", (None, True))
def test_updates_persist_append(tiled_websocket_context, envelope_format, persist):
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node using Tiled client
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_stream_immediate")
# Connect WebSocket using TestClient with msgpack format and authorization
with test_client.websocket_connect(
f"/api/v1/stream/single/test_stream_immediate?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Send 3 updates using Tiled client that append to the array
send_ws_updates(streaming_node, append_array, count=3, persist=persist)
# Receive and validate all updates
received = receive_ws_updates(websocket, count=3)
verify_ws_updates(received)
# Verify values of persisted data
if persist or persist is None:
# Combined effect of all sent values
expected_persisted = np.array(
[np.arange(10) + i for i in range(0, 4)]
).flatten()
else:
# Original values
expected_persisted = arr
persisted_data = streaming_node.read()
np.testing.assert_array_equal(persisted_data, expected_persisted)
def test_updates_append_without_persist(tiled_websocket_context):
context = tiled_websocket_context
client = from_context(context)
# Create streaming array node using Tiled client
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_stream_immediate")
with pytest.raises(ValueError, match="Cannot PATCH an array with both parameters"):
# Extending an array with persist=False is not yet supported
send_ws_updates(streaming_node, append_array, count=1, persist=False)
def test_close_stream_success(tiled_websocket_context):
"""Test successful close of an existing stream."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create a streaming array node
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_close_stream")
# Upload some data
streaming_node.write(np.arange(10) + 1)
# Add a small delay to ensure the stream is fully established
import time
time.sleep(0.5)
# Now close the stream
response = test_client.delete(
"/api/v1/stream/close/test_close_stream",
headers={"Authorization": "Apikey secret"},
)
assert response.status_code == 200
# Now close the stream again
response = test_client.delete(
"/api/v1/stream/close/test_close_stream",
headers={"Authorization": "Apikey secret"},
)
# close_stream is idempotent, so closing again should also return 200
assert response.status_code == 200
def test_close_stream_not_found(tiled_websocket_context):
"""Test close endpoint returns 404 for non-existent node."""
context = tiled_websocket_context
test_client = context.http_client
non_existent_node_id = "definitely_non_existent_node_99999999"
response = test_client.delete(
f"/api/v1/stream/close/{non_existent_node_id}",
headers={"Authorization": "Apikey secret"},
)
assert response.status_code == 404
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_connection_wrong_api_key(tiled_websocket_context, envelope_format):
"""Test websocket connection with wrong API key fails with 401."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node using correct key
arr = np.arange(10)
client.write_array(arr, key="test_auth_websocket")
# Try to connect to websocket with wrong API key
with pytest.raises(WebSocketDenialResponse) as exc_info:
with test_client.websocket_connect(
f"/api/v1/stream/single/test_auth_websocket?envelope_format={envelope_format}",
headers={"Authorization": "Apikey wrong_key"},
):
pass
assert exc_info.value.status_code == 401
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
@pytest.mark.parametrize(
"auth_message,expected_close_code",
[
pytest.param(
{"type": "auth", "api_key": "wrong_secret"},
4003,
id="wrong-api-key",
),
pytest.param(
{"type": "auth", "access_token": "invalid_token"},
4003,
id="invalid-access-token",
),
pytest.param(
{"type": "subscribe", "path": "foo"},
4001,
id="non-auth-message-type",
),
],
)
def test_first_message_auth_rejected(
tiled_websocket_context, envelope_format, auth_message, expected_close_code
):
"""First-message auth with bad credentials or wrong message type is rejected."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
arr = np.arange(10)
client.write_array(arr, key="test_first_msg_rejected")
context.api_key = None
with test_client.websocket_connect(
f"/api/v1/stream/single/test_first_msg_rejected?envelope_format={envelope_format}",
) as websocket:
websocket.send_json(auth_message)
msg = websocket.receive()
assert msg["type"] == "websocket.close"
assert msg.get("code") == expected_close_code
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_first_message_auth_with_api_key(tiled_websocket_context, envelope_format):
"""Test websocket first-message authentication with a valid API key."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node
arr = np.arange(10)
streaming_node = client.write_array(arr, key="test_first_msg_auth")
# Connect WITHOUT API key in headers — server will accept and wait for
# first-message auth.
with test_client.websocket_connect(
f"/api/v1/stream/single/test_first_msg_auth?envelope_format={envelope_format}",
) as websocket:
# Send first-message auth with valid API key
websocket.send_json({"type": "auth", "api_key": "secret"})
# Send an update and receive it, proving auth worked
send_ws_updates(streaming_node, overwrite_array, count=1)
received = receive_ws_updates(websocket, count=1)
verify_ws_updates(received)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_connection_public_no_api_key(tiled_websocket_context_public, envelope_format):
"""Test websocket connection to a public server with no API key works."""
context = tiled_websocket_context_public
client = from_context(context)
test_client = context.http_client
# Create streaming array node using correct key
arr = np.arange(10)
client.write_array(arr, key="test_auth_websocket")
# Strip API key so requests below are unauthenticated.
context.api_key = None
# Try to connect to (public) websocket with no API key
with test_client.websocket_connect(
f"/api/v1/stream/single/test_auth_websocket?envelope_format={envelope_format}",
):
pass
def test_close_stream_wrong_api_key(tiled_websocket_context):
"""Test close endpoint returns 403 with wrong API key."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
# Create streaming array node using correct key
arr = np.arange(10)
client.write_array(arr, key="test_auth_close")
# Try to close stream with wrong API key
response = test_client.delete(
"/api/v1/stream/close/test_auth_close",
headers={"Authorization": "Apikey wrong_key"},
)
assert response.status_code == 401
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_table_write_websocket(tiled_websocket_context, envelope_format):
"""Test that writing a full table triggers a WS event with the correct payload."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
df = pd.DataFrame({"label": ["a", "b"], "value": [1.0, 2.0]})
table_node = client.write_table(df, key="test_table_write")
with test_client.websocket_connect(
f"/api/v1/stream/single/test_table_write?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Overwrite with new data
df2 = pd.DataFrame({"label": ["c", "d"], "value": [3.0, 4.0]})
table_node.write_partition(0, df2)
# Receive schema + 1 data message
if envelope_format == "json":
schema_msg = websocket.receive_json()
else:
schema_msg = msgpack.unpackb(websocket.receive_bytes())
assert schema_msg["type"] == "table-schema"
assert "version" in schema_msg
# For JSON format, arrow_schema should have been converted to str
if envelope_format == "json":
assert isinstance(schema_msg["arrow_schema"], str)
if envelope_format == "json":
data_msg = websocket.receive_json()
else:
data_msg = msgpack.unpackb(websocket.receive_bytes())
assert data_msg["type"] == "table-data"
assert "timestamp" in data_msg
assert data_msg["append"] is False
if envelope_format == "json":
# stream_json transcodes Arrow IPC to dict-of-lists
payload = data_msg["payload"]
assert isinstance(payload, dict)
assert payload["label"] == ["c", "d"]
assert payload["value"] == [3.0, 4.0]
else:
# msgpack keeps the raw Arrow IPC bytes
assert isinstance(data_msg["payload"], bytes)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_table_append_websocket(tiled_websocket_context, envelope_format):
"""Test that appending rows to a table triggers a WS event."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
schema = pa.schema([("path", pa.string()), ("label", pa.string())])
table_node = client.create_appendable_table(schema, key="test_table_append")
with test_client.websocket_connect(
f"/api/v1/stream/single/test_table_append?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Append some rows (reset_index=False avoids __index_level_0__ mismatch)
df1 = pa.table({"path": ["/a/b"], "label": ["cat"]})
table_node.append_partition(0, df1)
df2 = pa.table({"path": ["/c/d", "/e/f"], "label": ["dog", "bird"]})
table_node.append_partition(0, df2)
# Receive schema + 2 data messages
if envelope_format == "json":
schema_msg = websocket.receive_json()
else:
schema_msg = msgpack.unpackb(websocket.receive_bytes())
assert schema_msg["type"] == "table-schema"
for expected_tbl in [df1, df2]:
if envelope_format == "json":
msg = websocket.receive_json()
else:
msg = msgpack.unpackb(websocket.receive_bytes())
assert msg["type"] == "table-data"
assert msg["append"] is True
if envelope_format == "json":
payload = msg["payload"]
assert payload["path"] == expected_tbl.column("path").to_pylist()
assert payload["label"] == expected_tbl.column("label").to_pylist()
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_table_multiple_appends_with_late_subscriber(
tiled_websocket_context, envelope_format
):
"""Late subscriber sees only new appends, not historical ones."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
schema = pa.schema([("x", pa.float64())])
table_node = client.create_appendable_table(schema, key="test_table_late_sub")
# Append before subscribing
table_node.append_partition(0, pa.table({"x": [1.0, 2.0]}))
with test_client.websocket_connect(
f"/api/v1/stream/single/test_table_late_sub?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Append after subscribing
table_node.append_partition(0, pa.table({"x": [3.0, 4.0]}))
# Receive schema
if envelope_format == "json":
schema_msg = websocket.receive_json()
else:
schema_msg = msgpack.unpackb(websocket.receive_bytes())
assert schema_msg["type"] == "table-schema"
# Receive the one new append
if envelope_format == "json":
msg = websocket.receive_json()
else:
msg = msgpack.unpackb(websocket.receive_bytes())
assert msg["type"] == "table-data"
assert msg["append"] is True
if envelope_format == "json":
assert msg["payload"]["x"] == [3.0, 4.0]
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_table_append_from_beginning(tiled_websocket_context, envelope_format):
"""Subscriber with start=0 replays historical table appends."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
schema = pa.schema([("val", pa.int64())])
table_node = client.create_appendable_table(schema, key="test_table_from_beginning")
# Append before subscribing
table_node.append_partition(0, pa.table({"val": [10, 20]}))
with test_client.websocket_connect(
f"/api/v1/stream/single/test_table_from_beginning?envelope_format={envelope_format}&start=0",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Schema
if envelope_format == "json":
schema_msg = websocket.receive_json()
else:
schema_msg = msgpack.unpackb(websocket.receive_bytes())
assert schema_msg["type"] == "table-schema"
# Should receive the historical append
if envelope_format == "json":
msg = websocket.receive_json()
else:
msg = msgpack.unpackb(websocket.receive_bytes())
assert msg["type"] == "table-data"
if envelope_format == "json":
assert msg["payload"]["val"] == [10, 20]
# Now append more and receive it
table_node.append_partition(0, pa.table({"val": [30]}))
if envelope_format == "json":
msg2 = websocket.receive_json()
else:
msg2 = msgpack.unpackb(websocket.receive_bytes())
assert msg2["type"] == "table-data"
if envelope_format == "json":
assert msg2["payload"]["val"] == [30]
def test_streaming_cache_config(tmp_path, redis_uri):
"Test streaming_cache config parsing"
config_path = tmp_path / "config.yml"
with open(config_path, "w") as file:
file.write(
f"""
trees:
- path: /
tree: catalog
args:
uri: "sqlite:///:memory:"
writable_storage:
- "file://localhost{str(tmp_path / "data")}"
- "duckdb:///{tmp_path / "data.duckdb"}"
init_if_not_exists: true
streaming_cache:
uri: "{redis_uri}"
data_ttl: 50
seq_ttl: 60
socket_timeout: 11
socket_connect_timeout: 12
"""
)
# Test that the config is parsed correctly.
config = parse_configs(config_path)
assert config.streaming_cache.uri == redis_uri
assert config.streaming_cache.data_ttl == 50
assert config.streaming_cache.seq_ttl == 60
assert config.streaming_cache.socket_timeout == 11
assert config.streaming_cache.socket_connect_timeout == 12
# On the standalone path the HA-only fields take their (off) defaults.
assert config.streaming_cache.sentinels is None
assert config.streaming_cache.service_name is None
assert config.streaming_cache.ssl is False
assert config.streaming_cache.health_check_interval == 10
def test_streaming_cache_sentinel_config(tmp_path):
"A Sentinel (HA) streaming_cache parses end-to-end, with uri left unset."
config_path = tmp_path / "config.yml"
with open(config_path, "w") as file:
file.write(
f"""
trees:
- path: /
tree: catalog
args:
uri: "sqlite:///:memory:"
writable_storage:
- "file://localhost{str(tmp_path / "data")}"
init_if_not_exists: true
streaming_cache:
sentinels:
- "redis-sentinel-1:26379"
- "redis-sentinel-2:26379"
service_name: "tiled-streaming-cache"
password: "secret"
ssl: true
"""
)
config = parse_configs(config_path)
assert config.streaming_cache.uri is None
assert config.streaming_cache.sentinels == [
"redis-sentinel-1:26379",
"redis-sentinel-2:26379",
]
assert config.streaming_cache.service_name == "tiled-streaming-cache"
assert config.streaming_cache.ssl is True
def _receive_schema(websocket, envelope_format):
"""Receive and return the schema message from a websocket."""
if envelope_format == "json":
msg = websocket.receive_json()
else:
msg = msgpack.unpackb(websocket.receive_bytes())
assert msg["type"] in ("array-schema", "table-schema")
return msg
@contextlib.contextmanager
def _unauthenticated(context):
"""Temporarily strip auth from context.http_client for unauthenticated requests.
Reuses the same TestClient (and its event loop) to avoid creating a second
event loop that would conflict with Redis connections created at startup.
"""
saved_auth = context.http_client.auth
context.http_client.auth = None
try:
yield context.http_client
finally:
context.http_client.auth = saved_auth
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_first_message_auth_with_api_key_multiuser(
tiled_websocket_context_multiuser, envelope_format
):
"""Test first-message auth with a valid API key in multi-user mode."""
context = tiled_websocket_context_multiuser
client = from_context(context)
key_info = context.create_api_key()
client.write_array(np.arange(10), key="test_apikey_multiuser")
with _unauthenticated(context) as test_client:
with test_client.websocket_connect(
f"/api/v1/stream/single/test_apikey_multiuser?envelope_format={envelope_format}",
) as websocket:
websocket.send_json({"type": "auth", "api_key": key_info["secret"]})
_receive_schema(websocket, envelope_format)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_first_message_auth_wrong_api_key_multiuser(
tiled_websocket_context_multiuser, envelope_format
):
"""Test first-message auth with a wrong API key in multi-user mode is rejected."""
context = tiled_websocket_context_multiuser
client = from_context(context)
client.write_array(np.arange(10), key="test_bad_apikey_multiuser")
with _unauthenticated(context) as test_client:
with test_client.websocket_connect(
f"/api/v1/stream/single/test_bad_apikey_multiuser?envelope_format={envelope_format}",
) as websocket:
websocket.send_json({"type": "auth", "api_key": "wrong-key"})
msg = websocket.receive()
assert msg["type"] == "websocket.close"
assert msg.get("code") == 4003
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_first_message_auth_with_access_token(
tiled_websocket_context_multiuser, envelope_format
):
"""Test websocket first-message authentication with a valid JWT access token."""
context = tiled_websocket_context_multiuser
client = from_context(context)
access_token = context.tokens["access_token"]
client.write_array(np.arange(10), key="test_jwt_first_msg")
with _unauthenticated(context) as test_client:
with test_client.websocket_connect(
f"/api/v1/stream/single/test_jwt_first_msg?envelope_format={envelope_format}",
) as websocket:
websocket.send_json({"type": "auth", "access_token": access_token})
_receive_schema(websocket, envelope_format)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_query_param_access_token(tiled_websocket_context_multiuser, envelope_format):
"""Test websocket connection with JWT access token as query parameter."""
context = tiled_websocket_context_multiuser
client = from_context(context)
access_token = context.tokens["access_token"]
client.write_array(np.arange(10), key="test_jwt_query_param")
token_param = urllib.parse.quote(access_token, safe="")
with _unauthenticated(context) as test_client:
with test_client.websocket_connect(
f"/api/v1/stream/single/test_jwt_query_param"
f"?envelope_format={envelope_format}&access_token={token_param}",
) as websocket:
_receive_schema(websocket, envelope_format)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_expired_access_token_query_param(
tiled_websocket_context_multiuser, envelope_format
):
"""Test that an expired JWT in the query parameter is rejected with 401."""
context = tiled_websocket_context_multiuser
client = from_context(context)
client.write_array(np.arange(10), key="test_expired_jwt")
# Create an expired token using the same secret key.
expired_token = jose.jwt.encode(
{
"sub": "fake",
"type": "access",
"exp": datetime.datetime.now(datetime.timezone.utc)
- datetime.timedelta(seconds=60),
},
"SECRET",
algorithm="HS256",
)
token_param = urllib.parse.quote(expired_token, safe="")
with _unauthenticated(context) as test_client:
with pytest.raises(WebSocketDenialResponse) as exc_info:
with test_client.websocket_connect(
f"/api/v1/stream/single/test_expired_jwt"
f"?envelope_format={envelope_format}&access_token={token_param}",
):
pass
assert exc_info.value.status_code == 401
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
@pytest.mark.parametrize(
"auth_message",
[
pytest.param({"type": "auth"}, id="no-credentials"),
pytest.param(
{"type": "auth", "access_token": "not.a.valid.jwt"},
id="invalid-jwt",
),
],
)
def test_first_message_jwt_auth_rejected(
tiled_websocket_context_multiuser, envelope_format, auth_message
):
"""First-message auth with missing or invalid JWT is rejected with close code 4003."""
context = tiled_websocket_context_multiuser
client = from_context(context)
client.write_array(np.arange(10), key="test_jwt_rejected")
with _unauthenticated(context) as test_client:
with test_client.websocket_connect(
f"/api/v1/stream/single/test_jwt_rejected?envelope_format={envelope_format}",
) as websocket:
websocket.send_json(auth_message)
msg = websocket.receive()
assert msg["type"] == "websocket.close"
assert msg.get("code") == 4003
# ---------------------------------------------------------------------------
# Ragged-array streaming
# ---------------------------------------------------------------------------
def _ragged_array(start=0):
"""Build a small jagged ragged.array. ``start`` shifts values for sequencing."""
import ragged
return ragged.array(
[
np.arange(2, dtype=np.int64) + start,
np.arange(3, dtype=np.int64) + start + 10,
]
)
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
@pytest.mark.parametrize("persist", (None, True, False))
def test_ragged_updates_persist_write(
tiled_websocket_context, envelope_format, persist
):
"""Stream a full-array overwrite of a ragged node."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
initial = _ragged_array(start=0)
streaming_node = client.write_ragged(initial, key="test_ragged_stream_write")
with test_client.websocket_connect(
f"/api/v1/stream/single/test_ragged_stream_write?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Send 2 overwriting updates
for i in range(1, 3):
kwargs = {} if persist is None else {"persist": persist}
streaming_node.write(_ragged_array(start=i), **kwargs)
# Receive schema + 2 updates
received = []
for _ in range(3):
if envelope_format == "json":
received.append(websocket.receive_json())
else:
received.append(msgpack.unpackb(websocket.receive_bytes()))
schema, *updates = received
assert "type" in schema and "version" in schema
assert schema["type"] == "ragged-schema"
assert len(updates) == 2
for msg in updates:
assert msg["type"] == "ragged-data"
assert "timestamp" in msg
assert "payload" in msg
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
@pytest.mark.parametrize("persist", (None, True, False))
def test_ragged_updates_persist_write_block(
tiled_websocket_context, envelope_format, persist
):
"""Stream a write_block update of a ragged node (single-chunk write)."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
initial = _ragged_array(start=0)
streaming_node = client.write_ragged(initial, key="test_ragged_stream_block")
with test_client.websocket_connect(
f"/api/v1/stream/single/test_ragged_stream_block?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# write_block with block=0 overwrites the only chunk
kwargs = {} if persist is None else {"persist": persist}
streaming_node.write_block(_ragged_array(start=1), block=0, **kwargs)
received = []
for _ in range(2): # schema + 1 update
if envelope_format == "json":
received.append(websocket.receive_json())
else:
received.append(msgpack.unpackb(websocket.receive_bytes()))
schema, update = received
assert schema["type"] == "ragged-schema"
assert update["type"] == "ragged-data"
assert "payload" in update
@pytest.mark.parametrize("envelope_format", (["msgpack", "json"]))
def test_ragged_updates_persist_append(tiled_websocket_context, envelope_format):
"""Stream a patch(extend=True) append of a ragged node."""
context = tiled_websocket_context
client = from_context(context)
test_client = context.http_client
initial = _ragged_array(start=0)
streaming_node = client.write_ragged(initial, key="test_ragged_stream_append")
with test_client.websocket_connect(
f"/api/v1/stream/single/test_ragged_stream_append?envelope_format={envelope_format}",
headers={"Authorization": "Apikey secret"},
) as websocket:
# Append a second chunk along the leftmost dimension
offset = streaming_node.shape[0]
streaming_node.patch(_ragged_array(start=1), offset=offset, extend=True)
received = []
for _ in range(2): # schema + 1 update
if envelope_format == "json":
received.append(websocket.receive_json())
else:
received.append(msgpack.unpackb(websocket.receive_bytes()))
schema, update = received
assert schema["type"] == "ragged-schema"
assert update["type"] == "ragged-data"
assert "payload" in update
# Persistence: the array should now have two chunks along the first dim.
assert streaming_node.shape[0] == 4
assert len(streaming_node.chunks[0]) == 2
import contextlib
import datetime
import sys
import urllib.parse
import dask.array
import jose.jwt
import msgpack
import numpy as np
import pandas as pd
import pyarrow as pa
import pytest
from starlette.testclient import WebSocketDenialResponse
from tiled.client import from_context
from tiled.config import parse_configs
pytestmark = pytest.mark.skipif(
sys.platform == "win32", reason="Requires Redis service"
)
def send_ws_updates(client, update_func, start=1, count=1, persist=None):
"""Helper to send updates via Tiled client in websocket tests."""
for i in range(start, start + count):
new_arr = np.arange(10) + i
update_func(client, new_arr, i, persist=persist)
# An update_func for send_ws_updates
def overwrite_array(client, new_arr, seq_num, persist=None):
_ = seq_num # seq_num is unused for these updates
client.write(new_arr, persist=persist)
# An update_func for send_ws_updates
def write_array_block(client, new_arr, seq_num, persist=None):
client.write_block(new_arr, block=(seq_num - 1, 0), persist=persist)
# An update_func for send_ws_updates
def patch_array(client, new_arr, seq_num, persist=None):
_ = seq_num # seq_num is unused for these updates
client.patch(new_arr, offset=(0,), persist=persist)
# An update_func for send_ws_updates
def append_array(client, new_arr, seq_num, persist=None):
client.patch(new_arr, offset=(10 * seq_num,), extend=True, persist=persist)
def receive_ws_updates(websocket, count=1):
"""Helper to receive updates in websocket tests."""
# Receive all updates
envelope_format = urllib.parse.parse_qs(websocket.scope["query_string"].decode())[
"envelope_format"
][0]
received = []
for _ in range(count + 1): # +1 for schema
if envelope_format == "json":
msg = websocket.receive_json()
else: # default to msgpack
msg_bytes = websocket.receive_bytes()
msg = msgpack.unpackb(msg_bytes)
received.append(msg)
# Verify all messages received (schema + n updates)
assert len(received) == count + 1
return received
def verify_ws_updates(received, start=1, chunked=False):
"""Verify that we received messages with the expected data"""
for i, msg in enumerate(received):
if i == 0: # schema
assert "type" in msg
assert "version" in msg
else:
assert "type" in msg
assert "timestamp" in msg
assert "payload" in msg
if chunked:
assert msg["shape"] == [1, 10]
else:
assert msg["shape"] == [10]
# Verify payload contains the expected array data
if isinstance(msg["payload"], bytes):
payload_array = np.frombuffer(msg["payload"], dtype=np.int64)
else:
payload_array = np.array(msg["payload"], dtype=np.int64)
expected_array = (np.arange(10) + (start - 1) + i).reshape(msg["shape"])
np.testing.assert_array_equal(
payload_array.reshape(msg["shape"]), expected_array
)
def test_subscribe_immediately_after_creation_websockets(
Xet Storage Details
- Size:
- 41.4 kB
- Xet hash:
- ecd246aada47779ee479134b9a8aaf73a7ff798b8e08cb0ff0fad893773ce9b8
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.