Buckets:
| import json | |
| import logging | |
| import os | |
| import threading | |
| import time | |
| from urllib.parse import urlparse | |
| import msgpack | |
| import numpy as np | |
| import websocket | |
| from locust import HttpUser, between, events, task | |
| # Configure logging | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| def _(parser): | |
| parser.add_argument( | |
| "--api-key", | |
| type=str, | |
| default="secret", | |
| help="API key for Tiled authentication (default: secret)", | |
| ) | |
| parser.add_argument( | |
| "--node-name", | |
| type=str, | |
| required=True, | |
| help="Node name for streaming test (required)", | |
| ) | |
| def on_locust_init(environment, **kwargs): | |
| if environment.host is None: | |
| raise ValueError( | |
| "Host must be specified with --host argument, or through the web-ui." | |
| ) | |
| # Create the streaming node once for all users | |
| create_streaming_node( | |
| environment.host, | |
| environment.parsed_options.api_key, | |
| environment.parsed_options.node_name, | |
| ) | |
| def create_streaming_node(host, api_key, node_name): | |
| """Create a streaming array node using Tiled client""" | |
| from tiled.client import from_uri | |
| # Connect to Tiled server using client | |
| client = from_uri(host, api_key=api_key) | |
| # Create initial streaming array | |
| arr = np.full(5, 0.0, dtype=np.float64) # Initial array with zeros | |
| client.write_array(arr, key=node_name) | |
| logger.info(f"Created streaming node: {node_name}") | |
| client.logout() | |
| class WriterUser(HttpUser): | |
| """User that writes streaming data to a Tiled node""" | |
| wait_time = between(0.1, 0.2) # Wait 0.1-0.2 seconds between writes | |
| weight = int(os.getenv("WRITER_WEIGHT", 1)) | |
| def on_start(self): | |
| """Initialize user state""" | |
| self.node_name = self.environment.parsed_options.node_name | |
| self.message_count = 0 | |
| self.api_key = self.environment.parsed_options.api_key | |
| # Set authentication header | |
| self.client.headers.update({"Authorization": f"Apikey {self.api_key}"}) | |
| # Run 10x as often as cleanup | |
| def write_data(self): | |
| """Write streaming data to the node""" | |
| # Create data with current timestamp as all values | |
| current_time = time.time() | |
| data = np.full(5, current_time, dtype=np.float64) | |
| binary_data = data.tobytes() | |
| # Post data to the streaming endpoint | |
| response = self.client.put( | |
| f"/api/v1/array/full/{self.node_name}", | |
| data=binary_data, | |
| headers={"Content-Type": "application/octet-stream"}, | |
| ) | |
| # Log status | |
| if response.status_code == 200: | |
| logger.debug(f"Wrote message {self.message_count} to node {self.node_name}") | |
| self.message_count += 1 | |
| else: | |
| logger.error( | |
| f"Failed to write message {self.message_count}: {response.status_code} - {response.text}" | |
| ) | |
| def cleanup(self): | |
| """Periodically cleanup the stream""" | |
| if self.message_count > 50: | |
| # Close the stream | |
| response = self.client.delete(f"/api/v1/stream/close/{self.node_name}") | |
| if response.status_code == 200: | |
| logger.info(f"Closed stream for node {self.node_name}") | |
| # Reset message count (node persists for other users) | |
| self.message_count = 0 | |
| class StreamingUser(HttpUser): | |
| """User that connects to websocket stream and measures latency""" | |
| wait_time = between(1, 2) | |
| weight = int(os.getenv("STREAMING_WEIGHT", 1)) | |
| def on_start(self): | |
| """Connect to the streaming endpoint""" | |
| self.node_name = self.environment.parsed_options.node_name | |
| self.api_key = self.environment.parsed_options.api_key | |
| self.envelope_format = "msgpack" # Use msgpack for efficiency | |
| self.ws = None | |
| self.connected = False | |
| # Set up authentication for HTTP requests | |
| self.client.headers.update({"Authorization": f"Apikey {self.api_key}"}) | |
| self._connect_websocket() | |
| def _connect_websocket(self): | |
| """Connect to the websocket stream""" | |
| try: | |
| # Parse host to get websocket URL | |
| parsed = urlparse(self.host) | |
| ws_scheme = "wss" if parsed.scheme == "https" else "ws" | |
| host = f"{ws_scheme}://{parsed.netloc}" | |
| ws_url = f"{host}/api/v1/stream/single/{self.node_name}?envelope_format={self.envelope_format}&start=0" | |
| # Create websocket connection | |
| self.ws = websocket.WebSocketApp( | |
| ws_url, | |
| header=[ | |
| f"Authorization: Apikey {self.api_key}" | |
| ], # Proper Apikey format for websockets | |
| on_open=self._on_open, | |
| on_message=self._on_message, | |
| on_error=self._on_error, | |
| on_close=self._on_close, | |
| ) | |
| # Start websocket in background thread | |
| self.ws_thread = threading.Thread(target=self.ws.run_forever) | |
| self.ws_thread.daemon = True | |
| self.ws_thread.start() | |
| # Wait a bit for connection | |
| time.sleep(0.5) | |
| except Exception as e: | |
| logger.error(f"Failed to connect websocket: {e}") | |
| def _on_open(self, ws): | |
| """Websocket connection opened""" | |
| self.connected = True | |
| logger.info(f"WebSocket connected to {self.node_name}") | |
| def _on_message(self, ws, message): | |
| """Process websocket messages and measure latency""" | |
| try: | |
| received_time = time.time() | |
| if isinstance(message, bytes): | |
| data = msgpack.unpackb(message) | |
| else: | |
| data = json.loads(message) | |
| # Extract timestamp from the payload (first element of the array) | |
| payload = data.get("payload") | |
| if payload and len(payload) > 0: | |
| # Convert bytes back to numpy array to get the timestamp | |
| payload_array = np.frombuffer(payload, dtype=np.float64) | |
| if len(payload_array) > 0: | |
| write_time = payload_array[0] | |
| latency_ms = (received_time - write_time) * 1000 | |
| logger.debug( | |
| f"WS latency (sequence {data.get('sequence', 'N/A')}): {latency_ms:.1f}ms" | |
| ) | |
| # Report to Locust | |
| events.request.fire( | |
| request_type="WS", | |
| name="write_to_websocket_delivery", | |
| response_time=latency_ms, | |
| response_length=len(message), | |
| exception=None, | |
| ) | |
| except Exception as e: | |
| logger.error(f"Error processing message: {e}") | |
| events.request.fire( | |
| request_type="WS", | |
| name="write_to_websocket_delivery", | |
| response_time=0, | |
| response_length=0, | |
| exception=e, | |
| ) | |
| def _on_error(self, ws, error): | |
| """Websocket error occurred""" | |
| logger.error(f"WebSocket error: {error}") | |
| self.connected = False | |
| def _on_close(self, ws, close_status_code, close_msg): | |
| """Websocket connection closed""" | |
| logger.info(f"WebSocket closed: {close_status_code} - {close_msg}") | |
| self.connected = False | |
| def keep_alive(self): | |
| """Dummy task to keep the user active while listening for messages""" | |
| if not self.connected and self.ws: | |
| # Try to reconnect if disconnected | |
| logger.info("Attempting to reconnect WebSocket...") | |
| self._connect_websocket() | |
| def on_stop(self): | |
| """Clean up websocket connection""" | |
| if self.ws: | |
| self.ws.close() | |
| logger.info("WebSocket connection closed") | |
Xet Storage Details
- Size:
- 7.99 kB
- Xet hash:
- 79a3b6f8092ec711d18afcc3a3bde5d2470c034f1367f43aa6563cff70a8ce00
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.