Buckets:
| import io | |
| import os | |
| import sys | |
| import tempfile | |
| from pathlib import Path | |
| from typing import Any | |
| from urllib.parse import urlparse | |
| import asyncpg | |
| import pytest | |
| import pytest_asyncio | |
| import stamina | |
| from sqlalchemy.ext.asyncio import create_async_engine | |
| from tiled import profiles | |
| from tiled.catalog import from_uri, in_memory | |
| from tiled.client import Context, from_context | |
| from tiled.client.base import BaseClient | |
| from tiled.config import Authentication | |
| from tiled.server.app import build_app | |
| from tiled.server.settings import get_settings | |
| from tiled.utils import ensure_specified_sql_driver | |
| from .utils import enter_username_password as utils_enter_uname_passwd | |
| from .utils import temp_postgres | |
| def deactivate_retries(): | |
| "Deactivate HTTP retries." | |
| stamina.set_active(False) | |
| def reset_settings_module(): | |
| """ | |
| Clear the cached Settings before module-scoped fixtures run. | |
| Without this, a previous module's build_app() can mutate the cached | |
| Settings singleton (e.g. setting database_settings.uri to a postgres URI), | |
| and module-scoped fixtures in the next module will see stale values | |
| because function-scoped cleanup hasn't run yet. This is critical under | |
| pytest-xdist where multiple modules share a worker process. | |
| """ | |
| get_settings.cache_clear() | |
| yield | |
| get_settings.cache_clear() | |
| def reset_settings(): | |
| """ | |
| Reset the FastAPI Settings. | |
| Fast API uses a global singleton for Settings. It is difficult to get | |
| around this, so our best option is to reset it. | |
| """ | |
| get_settings.cache_clear() | |
| yield | |
| def set_tiled_cache_dir(): | |
| """ | |
| Use a tmpdir instead of ~/.cache/tiled/tokens | |
| """ | |
| # Do not use tempdir pytest fixture because it would use the same tmpdir | |
| # as the one used by the test, and mix the files up. | |
| # Windows will not remove the directory while the http_response_cache.db | |
| # is still open. It is closed by transport shutdown, but not all tests | |
| # correctly shut down the transport. This is probably related to the | |
| # thread-leaking issue. | |
| # This option was added to TemporaryDirectory in Python 3.10 | |
| kwargs = {} | |
| if sys.platform.startswith("win") and sys.version_info >= (3, 10): | |
| kwargs["ignore_cleanup_errors"] = True | |
| with tempfile.TemporaryDirectory(**kwargs) as tmpdir: | |
| os.environ["TILED_CACHE_DIR"] = str(tmpdir) | |
| yield | |
| del os.environ["TILED_CACHE_DIR"] | |
| def buffer(): | |
| "Generate a temporary buffer for testing file export + re-import." | |
| with io.BytesIO() as buffer: | |
| yield buffer | |
| def buffer_factory(): | |
| buffers = [] | |
| def _buffer(): | |
| buf = io.BytesIO() | |
| buffers.append(buf) | |
| return buf | |
| yield _buffer | |
| for buf in buffers: | |
| buf.close() | |
| def tmp_profiles_dir(): | |
| """ | |
| Use a tmpdir instead of ~/.config/tiled/profiles | |
| """ | |
| # Do not use tempdir pytest fixture because it would use the same tmpdir | |
| # as the one used by the test, and mix the files up. | |
| with tempfile.TemporaryDirectory() as tmpdir: | |
| original = profiles.paths | |
| profiles.paths.clear() | |
| profiles.paths.extend([Path(tmpdir)]) | |
| profiles.load_profiles.cache_clear() | |
| yield | |
| profiles.paths.clear() | |
| profiles.paths.extend(original) | |
| def enter_username_password(): | |
| """ | |
| DEPRECATED: Use the normal (non-fixture) context manager in .utils. | |
| """ | |
| return utils_enter_uname_passwd | |
| def tmpdir_module(request, tmpdir_factory): | |
| """A tmpdir fixture for the module scope. Persists throughout the module.""" | |
| # Source: https://stackoverflow.com/a/31889843 | |
| return tmpdir_factory.mktemp(request.module.__name__) | |
| # Use this with pytest --log-cli-level=25 option. | |
| if os.getenv("TILED_DEBUG_LEAKED_THREADS"): | |
| import logging | |
| import threading | |
| import time | |
| def poll_enumerate(): | |
| logger = logging.getLogger(__name__) | |
| msg_level = int(logging.INFO + logging.WARNING) // 2 | |
| while True: | |
| time.sleep(1) | |
| logger.log(msg_level, "THREAD COUNT = %d", len(threading.enumerate())) | |
| thread = threading.Thread(target=poll_enumerate, daemon=True) | |
| thread.start() | |
| # To test with postgres, start a container like: | |
| # | |
| # docker run --name tiled-test-postgres -p 5432:5432 -e POSTGRES_PASSWORD=secret -d docker.io/postgres:16 | |
| # and set this env var like: | |
| # | |
| # TILED_TEST_POSTGRESQL_URI=postgresql://postgres:secret@localhost:5432 | |
| TILED_TEST_POSTGRESQL_URI = os.getenv("TILED_TEST_POSTGRESQL_URI") | |
| async def sqlite_uri(tmp_path: Path): | |
| yield f"sqlite:///{tmp_path}/tiled.sqlite" | |
| async def duckdb_uri(tmp_path: Path): | |
| yield f"duckdb:///{tmp_path}/tiled.duckdb" | |
| async def postgres_uri(): | |
| if not TILED_TEST_POSTGRESQL_URI: | |
| raise pytest.skip("No TILED_TEST_POSTGRESQL_URI configured") | |
| async with temp_postgres(TILED_TEST_POSTGRESQL_URI) as uri_with_database: | |
| yield uri_with_database | |
| def sqlite_or_postgres_uri(request): | |
| yield request.getfixturevalue(request.param) | |
| def sql_storage_uri(request): | |
| yield request.getfixturevalue(request.param) | |
| async def postgresql_catalog_adapter(request, tmpdir): | |
| """ | |
| Catalog adapter instance backed by PostgreSQL | |
| Note that startup() and shutdown() are not called, and must be run | |
| either manually (as in the fixture 'a') or via the app (as in the fixture 'client'). | |
| """ | |
| if not TILED_TEST_POSTGRESQL_URI: | |
| raise pytest.skip("No TILED_TEST_POSTGRESQL_URI configured") | |
| # Create temporary database. | |
| async with temp_postgres(TILED_TEST_POSTGRESQL_URI) as uri_with_database_name: | |
| # Build a catalog adapter on it, and initialize the database. | |
| yield from_uri( | |
| uri_with_database_name, | |
| writable_storage=str(tmpdir), | |
| init_if_not_exists=True, | |
| ) | |
| async def postgresql_with_example_data_adapter(request, tmpdir): | |
| """ | |
| Adapter instance | |
| Note that startup() and shutdown() are not called, and must be run | |
| either manually (as in the fixture 'a') or via the app (as in the fixture 'client'). | |
| """ | |
| if not TILED_TEST_POSTGRESQL_URI: | |
| raise pytest.skip("No TILED_TEST_POSTGRESQL_URI configured") | |
| DATABASE_NAME = "tiled-example-data" | |
| uri = TILED_TEST_POSTGRESQL_URI | |
| if uri.endswith("/"): | |
| uri = uri[:-1] | |
| uri_with_database_name = f"{uri}/{DATABASE_NAME}" | |
| engine = create_async_engine(ensure_specified_sql_driver(uri_with_database_name)) | |
| try: | |
| async with engine.connect(): | |
| pass | |
| except asyncpg.exceptions.InvalidCatalogNameError: | |
| raise pytest.skip( | |
| f"PostgreSQL instance contains no database named {DATABASE_NAME!r}" | |
| ) | |
| adapter = from_uri( | |
| uri_with_database_name, | |
| writable_storage=str(tmpdir), | |
| ) | |
| yield adapter | |
| async def sqlite_catalog_adapter(request, tmpdir): | |
| """ | |
| Catalog adapter instance backed by SQLite (in memory) | |
| Note that startup() and shutdown() are not called, and must be run | |
| either manually (as in the fixture 'a') or via the app (as in the fixture 'client'). | |
| """ | |
| yield in_memory(writable_storage=str(tmpdir)) | |
| async def sqlite_with_example_data_adapter(request, tmpdir): | |
| """ | |
| Adapter instance | |
| Note that startup() and shutdown() are not called, and must be run | |
| either manually (as in the fixture 'a') or via the app (as in the fixture 'client'). | |
| """ | |
| SQLITE_DATABASE_PATH = Path("./tiled_test_db_sqlite.db") | |
| if not SQLITE_DATABASE_PATH.is_file(): | |
| raise pytest.skip(f"Could not find {SQLITE_DATABASE_PATH}") | |
| adapter = from_uri( | |
| f"sqlite:///{SQLITE_DATABASE_PATH}", | |
| writable_storage=str(tmpdir), | |
| ) | |
| yield adapter | |
| def catalog_adapter(request): | |
| """ | |
| Catalog adapter instance | |
| Note that startup() and shutdown() are not called, and must be run | |
| either manually (as in the fixture 'a') or via the app (as in the fixture 'client'). | |
| """ | |
| yield request.getfixturevalue(request.param) | |
| def example_data_adapter(request): | |
| """ | |
| Adapter instance | |
| Note that startup() and shutdown() are not called, and must be run | |
| either manually (as in the fixture 'a') or via the app (as in the fixture 'client'). | |
| """ | |
| yield request.getfixturevalue(request.param) | |
| def set_and_deplete(request: pytest.FixtureRequest): | |
| "Initial values must all be removed or else calling tests fail" | |
| values = set(request.param) | |
| yield values | |
| assert not values | |
| def url_limit(request: pytest.FixtureRequest): | |
| """Adjust the URL length limit for the client's GET requests. | |
| Data will be fetched by GET requests when the URL_CHARACTER_LIMIT is long, | |
| and by equivalent POST requests when the URL_CHARACTER_LIMIT is short. | |
| """ | |
| URL_CHARACTER_LIMIT = int(request.param) | |
| PREVIOUS_LIMIT = BaseClient.URL_CHARACTER_LIMIT | |
| # Temporarily adjust the URL length limit to change client behavior. | |
| BaseClient.URL_CHARACTER_LIMIT = URL_CHARACTER_LIMIT | |
| yield | |
| # Then restore the original value. | |
| BaseClient.URL_CHARACTER_LIMIT = PREVIOUS_LIMIT | |
| def redis_uri(): | |
| if uri := os.getenv("TILED_TEST_REDIS"): | |
| import redis | |
| client = redis.from_url(uri, socket_timeout=10, socket_connect_timeout=30) | |
| # Delete all keys from the current database before and after test. | |
| client.flushdb() | |
| yield uri | |
| client.flushdb() | |
| else: | |
| raise pytest.skip("No TILED_TEST_REDIS configured") | |
| def minio_uri(): | |
| if uri := os.getenv("TILED_TEST_BUCKET"): | |
| from minio import Minio | |
| from minio.deleteobjects import DeleteObject | |
| # For convenience, we split the bucket from a string | |
| url = urlparse(uri) | |
| bucket_name = url.path.lstrip("/") | |
| uri = url._replace(netloc="{}:{}".format(url.hostname, url.port), path="") | |
| client = Minio( | |
| endpoint=uri.geturl(), | |
| access_key=url.username, | |
| secret_key=url.password, | |
| secure=False, | |
| ) | |
| # Reset the state of the bucket after each test. | |
| if client.bucket_exists(bucket_name=bucket_name): | |
| delete_object_list = map( | |
| lambda x: DeleteObject(object_name=x.object_name), | |
| client.list_objects(bucket_name=bucket_name, recursive=True), | |
| ) | |
| errors = client.remove_objects( | |
| bucket_name=bucket_name, delete_object_list=delete_object_list | |
| ) | |
| for error in errors: | |
| print("error occurred when deleting object", error) | |
| else: | |
| client.make_bucket(bucket_name=bucket_name) | |
| else: | |
| raise pytest.skip("No TILED_TEST_BUCKET configured") | |
| # Shared authentication config for tests that need multi-user JWT auth. | |
| # Uses the toy DictionaryAuthenticator with alice/bob users. | |
| TOY_AUTHENTICATION = { | |
| "secret_keys": ["SECRET"], | |
| "providers": [ | |
| { | |
| "provider": "toy", | |
| "authenticator": "tiled.authenticators:DictionaryAuthenticator", | |
| "args": { | |
| "users_to_passwords": {"alice": "secret1", "bob": "secret2"}, | |
| }, | |
| } | |
| ], | |
| } | |
| def init_auth_database(tmpdir): | |
| """Initialize an auth database and return its URI.""" | |
| import subprocess | |
| database_uri = f"sqlite:///{tmpdir / 'auth.db'}" | |
| subprocess.run( | |
| [sys.executable, "-m", "tiled", "admin", "initialize-database", database_uri], | |
| check=True, | |
| capture_output=True, | |
| ) | |
| return database_uri | |
| def build_test_app(tmpdir, redis_uri, public=False): | |
| tree = from_uri( | |
| "sqlite:///:memory:", | |
| writable_storage=[ | |
| f"file://localhost{str(tmpdir / 'data')}", | |
| f"duckdb:///{tmpdir / 'data.duckdb'}", | |
| ], | |
| readable_storage=[Path(tempfile.gettempdir()).resolve()], | |
| init_if_not_exists=True, | |
| # This uses shorter defaults than the production defaults. Nothing in | |
| # the test suite should be going on for more than ten minutes. | |
| cache_config={ | |
| "uri": redis_uri, | |
| "data_ttl": 600, # 10 minutes | |
| "seq_ttl": 600, # 10 minutes | |
| "socket_timeout": 600, # 10 minutes | |
| "socket_connect_timeout": 10, | |
| }, | |
| ) | |
| app = build_app( | |
| tree, | |
| authentication=Authentication( | |
| single_user_api_key="secret", | |
| allow_anonymous_access=public, | |
| ), | |
| ) | |
| return app | |
| def tiled_websocket_context(tmpdir, redis_uri): | |
| """Fixture that provides a Tiled context with websocket support.""" | |
| with Context.from_app(build_test_app(tmpdir, redis_uri, public=False)) as context: | |
| yield context | |
| def tiled_websocket_context_public(tmpdir, redis_uri): | |
| """Fixture that provides a Tiled context with websocket support.""" | |
| with Context.from_app(build_test_app(tmpdir, redis_uri, public=True)) as context: | |
| yield context | |
| def tiled_websocket_context_multiuser(tmpdir, redis_uri, enter_username_password): | |
| """Fixture that provides a multi-user Tiled context with JWT auth and websocket support. | |
| Logs in as 'alice' during setup so that context.tokens and from_context() | |
| work without additional login steps in the test. | |
| """ | |
| from tiled.server.app import build_app_from_config | |
| database_uri = init_auth_database(tmpdir) | |
| config = { | |
| "authentication": TOY_AUTHENTICATION, | |
| "database": {"uri": database_uri}, | |
| "trees": [ | |
| { | |
| "tree": "catalog", | |
| "path": "/", | |
| "args": { | |
| "uri": "sqlite:///:memory:", | |
| "writable_storage": [str(tmpdir / "data")], | |
| "init_if_not_exists": True, | |
| }, | |
| }, | |
| ], | |
| "streaming_cache": { | |
| "uri": redis_uri, | |
| "data_ttl": 600, | |
| "seq_ttl": 600, | |
| "socket_timeout": 600, | |
| "socket_connect_timeout": 10, | |
| }, | |
| } | |
| app = build_app_from_config(config) | |
| with Context.from_app(app) as context: | |
| with enter_username_password("alice", "secret1"): | |
| from_context(context) # triggers login, caches tokens | |
| yield context | |
| def base_url() -> str: | |
| return "https://example.com/realms/example" | |
| def well_known_response(base_url: str) -> dict[str, Any]: | |
| return { | |
| "id_token_signing_alg_values_supported": ["RS256"], | |
| "issuer": base_url, | |
| "jwks_uri": f"{base_url}protocol/openid-connect/certs", | |
| "authorization_endpoint": f"{base_url}protocol/openid-connect/auth", | |
| "token_endpoint": f"{base_url}protocol/openid-connect/token", | |
| "device_authorization_endpoint": f"{base_url}protocol/openid-connect/auth/device", | |
| "end_session_endpoint": f"{base_url}protocol/openid-connect/logout", | |
| } | |
Xet Storage Details
- Size:
- 15.8 kB
- Xet hash:
- fef1b39e48984f892d6de50d29778963948620cbe55c03c51b92433921c93ff9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.