Buckets:
| import asyncio | |
| import string | |
| import subprocess | |
| import sys | |
| from contextlib import nullcontext | |
| import numpy | |
| import pytest | |
| import pytest_asyncio | |
| from starlette.status import HTTP_400_BAD_REQUEST | |
| from tiled.adapters.array import ArrayAdapter | |
| from tiled.adapters.mapping import MapAdapter | |
| from tiled.catalog import from_uri, in_memory | |
| from tiled.client import Context, from_context | |
| from tiled.queries import ( | |
| Comparison, | |
| Contains, | |
| Eq, | |
| FullText, | |
| In, | |
| Key, | |
| KeyPresent, | |
| KeysFilter, | |
| Like, | |
| NotEq, | |
| NotIn, | |
| Regex, | |
| SpecsQuery, | |
| StructureFamilyQuery, | |
| ) | |
| from tiled.server.app import build_app | |
| from .conftest import TILED_TEST_POSTGRESQL_URI | |
| from .utils import fail_with_status_code, sqlite_from_dump, temp_postgres | |
| keys = list(string.ascii_lowercase) | |
| mapping = { | |
| letter: ArrayAdapter.from_array( | |
| number * numpy.ones(10), | |
| metadata={"letter": letter, "number": number}, | |
| specs=[letter], | |
| ) | |
| for letter, number in zip(keys, range(26)) | |
| } | |
| mapping["does_contain_z"] = ArrayAdapter.from_array( | |
| numpy.ones(10), metadata={"letters": list(string.ascii_lowercase)} | |
| ) | |
| mapping["does_not_contain_z"] = ArrayAdapter.from_array( | |
| numpy.ones(10), metadata={"letters": list(string.ascii_lowercase[:-1])} | |
| ) | |
| mapping["full_text_test_case"] = ArrayAdapter.from_array( | |
| numpy.ones(10), metadata={"color": "purple"} | |
| ) | |
| mapping["full_text_test_case_urple"] = ArrayAdapter.from_array( | |
| numpy.ones(10), metadata={"color": "urple"} | |
| ) | |
| mapping["specs_foo_bar"] = ArrayAdapter.from_array(numpy.ones(10), specs=["foo", "bar"]) | |
| mapping["specs_foo_bar_baz"] = ArrayAdapter.from_array( | |
| numpy.ones(10), specs=["foo", "bar", "baz"] | |
| ) | |
| nested_metadata = { | |
| "nested": {"nested-key-1": "nested-value-1", "nested-key-2": "nested-value-2"} | |
| } | |
| mapping["nested_key_test"] = ArrayAdapter.from_array( | |
| numpy.ones(10), metadata=nested_metadata | |
| ) | |
| def event_loop(): | |
| # https://stackoverflow.com/a/56238383/1221924 | |
| loop = asyncio.new_event_loop() | |
| yield loop | |
| loop.close() | |
| async def client(request, tmpdir_module): | |
| if request.param == "map": | |
| tree = MapAdapter(mapping, metadata={"backend": request.param}) | |
| app = build_app(tree) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| yield client | |
| elif request.param == "sqlite": | |
| tree = in_memory( | |
| writable_storage=str(tmpdir_module / "sqlite"), | |
| metadata={"backend": request.param}, | |
| ) | |
| app = build_app(tree) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| for k, v in mapping.items(): | |
| client.write_array( | |
| v.read(), key=k, metadata=dict(v.metadata()), specs=v.specs | |
| ) | |
| yield client | |
| elif request.param == "postgresql": | |
| if not TILED_TEST_POSTGRESQL_URI: | |
| raise pytest.skip("No TILED_TEST_POSTGRESQL_URI configured") | |
| # Create temporary database. | |
| async with temp_postgres(TILED_TEST_POSTGRESQL_URI) as uri_with_database_name: | |
| subprocess.run( | |
| [ | |
| sys.executable, | |
| "-m", | |
| "tiled", | |
| "catalog", | |
| "init", | |
| uri_with_database_name, | |
| ], | |
| check=True, | |
| capture_output=True, | |
| ) | |
| tree = from_uri( | |
| uri_with_database_name, | |
| writable_storage=str(tmpdir_module / "postgresql"), | |
| metadata={"backend": request.param}, | |
| ) | |
| app = build_app(tree) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| # Write data into catalog. | |
| for k, v in mapping.items(): | |
| client.write_array( | |
| v.read(), | |
| key=k, | |
| metadata=dict(v.metadata()), | |
| specs=v.specs, | |
| ) | |
| yield client | |
| else: | |
| assert False | |
| def test_key(client): | |
| "Binary operators with Key create query objects." | |
| assert (Key("a") == 1) == Eq("a", 1) | |
| assert (Key("a") != 2) == NotEq("a", 2) | |
| assert (Key("a") > 1) == Comparison("gt", "a", 1) | |
| assert (Key("a") < 1) == Comparison("lt", "a", 1) | |
| assert (Key("a") >= 1) == Comparison("ge", "a", 1) | |
| assert (Key("a") <= 1) == Comparison("le", "a", 1) | |
| def test_eq(client): | |
| # Test encoding letters and ints. | |
| assert list(client.search(Eq("letter", "a"))) == ["a"] | |
| assert list(client.search(Eq("letter", "b"))) == ["b"] | |
| assert list(client.search(Eq("number", 0))) == ["a"] | |
| assert list(client.search(Eq("number", 1))) == ["b"] | |
| # Number is an int, not a string, so this should not match anything. | |
| assert list(client.search(Eq("number", "0"))) == [] | |
| def test_noteq(client): | |
| # Test encoding letters and ints. | |
| assert list(client.search(NotEq("letter", "a"))) != ["a"] | |
| assert list(client.search(NotEq("letter", "b"))) != ["b"] | |
| assert list(client.search(NotEq("number", 0))) != ["a"] | |
| assert list(client.search(NotEq("number", 1))) != ["b"] | |
| def test_comparison(client): | |
| assert list(client.search(Comparison("gt", "number", 24))) == ["z"] | |
| assert list(client.search(Comparison("ge", "number", 24))) == ["y", "z"] | |
| assert list(client.search(Comparison("lt", "number", 1))) == ["a"] | |
| assert list(client.search(Comparison("le", "number", 1))) == ["a", "b"] | |
| def test_contains(client): | |
| assert list(client.search(Contains("letters", "z"))) == ["does_contain_z"] | |
| def test_full_text(client): | |
| "Basic test of FullText query" | |
| assert list(client.search(FullText("z"))) == ["z", "does_contain_z"] | |
| # plainto_tsquery fails to find certain words, weirdly, so it is a useful | |
| # test that we are using tsquery | |
| assert list(client.search(FullText("purple"))) == ["full_text_test_case"] | |
| assert list(client.search(FullText("urple"))) == ["full_text_test_case_urple"] | |
| def test_full_text_after_migration(): | |
| # Load a SQL database created by an older version of Tiled, predating FullText | |
| # support, and verify that the migration indexes the pre-existing metadata. | |
| with sqlite_from_dump("before_creating_fts5_virtual_table.sql") as database_path: | |
| subprocess.check_call( | |
| [sys.executable] | |
| + f"-m tiled catalog upgrade-database sqlite:///{database_path}".split() | |
| ) | |
| catalog = from_uri(database_path) | |
| app = build_app(catalog) | |
| with Context.from_app(app) as context: | |
| client = from_context(context) | |
| assert list(client.search(FullText("blue"))) == ["x"] | |
| assert list(client.search(FullText("red"))) == [] # does not exist | |
| def test_full_text_update(client): | |
| if client.metadata["backend"] == "map": | |
| pytest.skip("Updating not supported") | |
| # Update the fulltext index and check that it is current with the main data. | |
| try: | |
| client["full_text_test_case"].update_metadata({"color": "red"}) | |
| assert list(client.search(FullText("purple"))) == [] | |
| assert list(client.search(FullText("red"))) == ["full_text_test_case"] | |
| finally: | |
| # Reset case in the event tests are run out of order. | |
| client["full_text_test_case"].update_metadata({"color": "purple"}) | |
| def test_full_text_delete(client): | |
| if client.metadata["backend"] == "map": | |
| pytest.skip("Updating not supported") | |
| # Delete a record the fulltext index and check that it is current with the main data. | |
| client.write_array(numpy.ones(10), metadata={"item": "toaster"}, key="test_delete") | |
| # Assert that the data was written | |
| assert list(client.search(FullText("toaster"))) == ["test_delete"] | |
| client.delete_contents("test_delete", external_only=False) | |
| assert list(client.search(FullText("purple"))) == ["full_text_test_case"] | |
| assert list(client.search(FullText("toaster"))) == [] | |
| def test_regex(client): | |
| if client.metadata["backend"] in {"postgresql", "sqlite"}: | |
| def cm(): | |
| return fail_with_status_code(HTTP_400_BAD_REQUEST) | |
| else: | |
| cm = nullcontext | |
| with cm(): | |
| assert list(client.search(Regex("letter", "^z$"))) == ["z"] | |
| with cm(): | |
| assert ( | |
| list(client.search(Regex("letter", "^Z$"))) == [] | |
| ) # default case_sensitive=True | |
| with cm(): | |
| assert list(client.search(Regex("letter", "^Z$", case_sensitive=False))) == [ | |
| "z" | |
| ] | |
| with cm(): | |
| assert list(client.search(Regex("letter", "^Z$", case_sensitive=True))) == [] | |
| with cm(): | |
| assert list(client.search(Regex("letter", "[a-c]"))) == ["a", "b", "c"] | |
| with cm(): | |
| # Check that if the key is not a string it is ignored. | |
| assert list(client.search(Regex("letters", "anything"))) == [] | |
| def test_not_and_and_or(client): | |
| with pytest.raises(TypeError): | |
| not (Key("color") == "red") # type: ignore | |
| with pytest.raises(TypeError): | |
| (Key("color") == "red") and (Key("sample") == "Ni") # type: ignore | |
| with pytest.raises(TypeError): | |
| (Key("color") == "red") or (Key("sample") == "Ni") # type: ignore | |
| def test_in(client, query_values): | |
| assert sorted(list(client.search(In("letter", query_values)))) == [ | |
| "a", | |
| "k", | |
| "z", | |
| ] | |
| def test_in_empty(client): | |
| assert list(client.search(In("letter", []))) == [] | |
| def test_notin(client, query_values): | |
| # TODO: Postgres and SQlite ACTUALLY treat this query differently in external testing. | |
| # SQLite WILL NOT include fields that do not have the key, which is correct. | |
| # Postgres WILL include fields that do not have the key, | |
| # because by extension they do not have the value. Also correct. Why? | |
| assert sorted(list(client.search(NotIn("letter", query_values)))) == sorted( | |
| list( | |
| set( | |
| list(mapping.keys()) | |
| if client.metadata["backend"] == "postgresql" | |
| else keys | |
| ) | |
| - set(["a", "k", "z"]) | |
| ) | |
| ) | |
| def test_not_in_empty(client): | |
| assert sorted(list(client.search(NotIn("letter", [])))) == sorted(list(client)) | |
| def test_specs(client, include_values, exclude_values): | |
| assert list( | |
| client.search(SpecsQuery(include=include_values, exclude=exclude_values)) | |
| ) == ["specs_foo_bar"] | |
| def test_structure_families(client): | |
| with pytest.raises(ValueError): | |
| StructureFamilyQuery("foo") | |
| assert set(client.search(StructureFamilyQuery("array"))) == set(mapping) | |
| def test_key_present(client): | |
| if client.metadata["backend"] == "map": | |
| pytest.skip("No 'KeyPresent' support on MapAdapter") | |
| # These containers have a "color" key. | |
| assert list(client.search(KeyPresent("color"))) == [ | |
| "full_text_test_case", | |
| "full_text_test_case_urple", | |
| ] | |
| # outer key present | |
| assert list(client.search(KeyPresent("nested"))) == ["nested_key_test"] | |
| # one of the inner keys present | |
| assert list(client.search(KeyPresent("nested.nested-key-1"))) == ["nested_key_test"] | |
| assert list(client.search(KeyPresent("nested.nested-key-2"))) == ["nested_key_test"] | |
| # both inner keys | |
| assert list( | |
| client.search(KeyPresent("nested.nested-key-1")).search( | |
| KeyPresent("nested.nested-key-2") | |
| ) | |
| ) == ["nested_key_test"] | |
| # inner key not present | |
| assert list(client.search(KeyPresent("nested.nested-key-3"))) == [] | |
| # outer key not present | |
| assert list(client.search(KeyPresent("nonsense.nested-key-1"))) == [] | |
| # These are all the containers that do not have a "number" key. | |
| assert list(client.search(KeyPresent("number", False))) == [ | |
| "does_contain_z", | |
| "does_not_contain_z", | |
| "full_text_test_case", | |
| "full_text_test_case_urple", | |
| "specs_foo_bar", | |
| "specs_foo_bar_baz", | |
| "nested_key_test", | |
| ] | |
| def test_keys_filter(client): | |
| expected = ["a", "b", "c"] | |
| results = client.search(KeysFilter(keys=expected)) | |
| assert set(results) == set(expected) | |
| def test_like(client): | |
| if client.metadata["backend"] == "map": | |
| pytest.skip("No 'LIKE' support on MapAdapter") | |
| expected = ["full_text_test_case", "full_text_test_case_urple"] | |
| results = client.search(Like("color", "%urple")) | |
| assert set(results) == set(expected) | |
Xet Storage Details
- Size:
- 13.2 kB
- Xet hash:
- 461b4a3af7a4a5bb58be4dda62eee7e613d05fc00a9eb57b9527b471861eb565
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.