#  # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with  # the License. You may obtain a copy of the License at  #  #        http://www.apache.org/licenses/LICENSE-2.0  #  # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on  # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the  # specific language governing permissions and limitations under the License.  import os  import pathlib    import pytest    from src.taipy.core._version._version_manager import _VersionManager  from src.taipy.core.config.data_node_config import DataNodeConfig  from src.taipy.core.data._data_manager import _DataManager  from src.taipy.core.data.csv import CSVDataNode  from src.taipy.core.data.data_node_id import DataNodeId  from src.taipy.core.data.in_memory import InMemoryDataNode  from src.taipy.core.data.pickle import PickleDataNode  from src.taipy.core.exceptions.exceptions import InvalidDataNodeType, ModelNotFound  from taipy.config.common.scope import Scope  from taipy.config.config import Config  from tests.core.utils.named_temporary_file import NamedTemporaryFile      def file_exists(file_path: str) -> bool:      return os.path.exists(file_path)      class TestDataManager:      def test_create_data_node_and_modify_properties_does_not_modify_config(self):          dn_config = Config.configure_data_node(id="name", foo="bar")          dn = _DataManager._create_and_set(dn_config, None, None)          assert dn_config.properties.get("foo") == "bar"          assert dn_config.properties.get("baz") is None            dn.properties["baz"] = "qux"          _DataManager._set(dn)          assert dn_config.properties.get("foo") == "bar"          assert dn_config.properties.get("baz") is None          assert dn.properties.get("foo") == "bar"          assert dn.properties.get("baz") == "qux"        def test_create_data_node_with_name_provided(self):          dn_config = Config.configure_data_node(id="dn", foo="bar", name="acb")          dn = _DataManager._create_and_set(dn_config, None, None)          assert dn.name == "acb"        def test_create_and_get_csv_data_node(self):          # Test we can instantiate a CsvDataNode from DataNodeConfig with :          # - a csv type          # - a default scenario scope          # - No owner_id          csv_dn_config = Config.configure_data_node(id="foo", storage_type="csv", path="bar", has_header=True)          csv_dn = _DataManager._create_and_set(csv_dn_config, None, None)            assert isinstance(csv_dn, CSVDataNode)          assert isinstance(_DataManager._get(csv_dn.id), CSVDataNode)            assert _DataManager._exists(csv_dn.id)          assert _DataManager._get(csv_dn.id) is not None          assert _DataManager._get(csv_dn.id).id == csv_dn.id          assert _DataManager._get(csv_dn.id).config_id == "foo"          assert _DataManager._get(csv_dn.id).config_id == csv_dn.config_id          assert _DataManager._get(csv_dn.id).scope == Scope.SCENARIO          assert _DataManager._get(csv_dn.id).scope == csv_dn.scope          assert _DataManager._get(csv_dn.id).owner_id is None          assert _DataManager._get(csv_dn.id).owner_id == csv_dn.owner_id          assert _DataManager._get(csv_dn.id).parent_ids == set()          assert _DataManager._get(csv_dn.id).parent_ids == csv_dn.parent_ids          assert _DataManager._get(csv_dn.id).last_edit_date is None          assert _DataManager._get(csv_dn.id).last_edit_date == csv_dn.last_edit_date          assert _DataManager._get(csv_dn.id).job_ids == []          assert _DataManager._get(csv_dn.id).job_ids == csv_dn.job_ids          assert not _DataManager._get(csv_dn.id).is_ready_for_reading          assert _DataManager._get(csv_dn.id).is_ready_for_reading == csv_dn.is_ready_for_reading          assert len(_DataManager._get(csv_dn.id).properties) == 4          assert _DataManager._get(csv_dn.id).properties.get("path") == "bar"          assert _DataManager._get(csv_dn.id).properties.get("encoding") == "utf-8"          assert _DataManager._get(csv_dn.id).properties.get("has_header") is True          assert _DataManager._get(csv_dn.id).properties.get("exposed_type") == "pandas"          assert _DataManager._get(csv_dn.id).properties == csv_dn.properties          assert _DataManager._get(csv_dn.id).edit_in_progress is False          assert _DataManager._get(csv_dn.id)._editor_id is None          assert _DataManager._get(csv_dn.id)._editor_expiration_date is None            assert _DataManager._get(csv_dn) is not None          assert _DataManager._get(csv_dn).id == csv_dn.id          assert _DataManager._get(csv_dn).config_id == "foo"          assert _DataManager._get(csv_dn).config_id == csv_dn.config_id          assert _DataManager._get(csv_dn).scope == Scope.SCENARIO          assert _DataManager._get(csv_dn).scope == csv_dn.scope          assert _DataManager._get(csv_dn).owner_id is None          assert _DataManager._get(csv_dn).owner_id == csv_dn.owner_id          assert _DataManager._get(csv_dn).parent_ids == set()          assert _DataManager._get(csv_dn).parent_ids == csv_dn.parent_ids          assert _DataManager._get(csv_dn).last_edit_date is None          assert _DataManager._get(csv_dn).last_edit_date == csv_dn.last_edit_date          assert _DataManager._get(csv_dn).job_ids == []          assert _DataManager._get(csv_dn).job_ids == csv_dn.job_ids          assert not _DataManager._get(csv_dn).is_ready_for_reading          assert _DataManager._get(csv_dn).is_ready_for_reading == csv_dn.is_ready_for_reading          assert len(_DataManager._get(csv_dn).properties) == 4          assert _DataManager._get(csv_dn).properties.get("path") == "bar"          assert _DataManager._get(csv_dn).properties.get("encoding") == "utf-8"          assert _DataManager._get(csv_dn).properties.get("has_header") is True          assert _DataManager._get(csv_dn.id).properties.get("exposed_type") == "pandas"          assert _DataManager._get(csv_dn).properties == csv_dn.properties          assert _DataManager._get(csv_dn.id).edit_in_progress is False          assert _DataManager._get(csv_dn.id)._editor_id is None          assert _DataManager._get(csv_dn.id)._editor_expiration_date is None        def test_edit_and_get_data_node(self):          config = Config.configure_pickle_data_node(id="foo")          dn = _DataManager._create_and_set(config, None, None)            assert _DataManager._get(dn.id).last_edit_date is None          assert len(_DataManager._get(dn.id).properties) == 1          assert _DataManager._get(dn.id).properties.get("is_generated")          assert not _DataManager._get(dn.id).edit_in_progress          assert _DataManager._get(dn.id)._editor_id is None          assert _DataManager._get(dn.id)._editor_expiration_date is None            dn.lock_edit("foo")            assert _DataManager._get(dn.id).last_edit_date is None          assert len(_DataManager._get(dn.id).properties) == 1          assert _DataManager._get(dn.id).properties.get("is_generated")          assert _DataManager._get(dn.id).edit_in_progress          assert _DataManager._get(dn.id).editor_id == "foo"          assert _DataManager._get(dn.id).editor_expiration_date is not None            dn.unlock_edit("foo")            assert _DataManager._get(dn.id).last_edit_date is None          assert len(_DataManager._get(dn.id).properties) == 1          assert _DataManager._get(dn.id).properties.get("is_generated")          assert not _DataManager._get(dn.id).edit_in_progress          assert _DataManager._get(dn.id).editor_id is None          assert _DataManager._get(dn.id).editor_expiration_date is None        def test_create_and_get_in_memory_data_node(self):          # Test we can instantiate an InMemoryDataNode from DataNodeConfig with :          # - an in_memory type          # - a scenario scope          # - an owner id          # - some default data          in_memory_dn_config = Config.configure_data_node(              id="baz", storage_type="in_memory", scope=Scope.SCENARIO, default_data="qux", other_data="foo"          )          in_mem_dn = _DataManager._create_and_set(in_memory_dn_config, "Scenario_id", {"task_id"})            assert isinstance(in_mem_dn, InMemoryDataNode)          assert isinstance(_DataManager._get(in_mem_dn.id), InMemoryDataNode)            assert _DataManager._exists(in_mem_dn.id)          assert _DataManager._get(in_mem_dn.id) is not None          assert _DataManager._get(in_mem_dn.id).id == in_mem_dn.id          assert _DataManager._get(in_mem_dn.id).config_id == "baz"          assert _DataManager._get(in_mem_dn.id).config_id == in_mem_dn.config_id          assert _DataManager._get(in_mem_dn.id).scope == Scope.SCENARIO          assert _DataManager._get(in_mem_dn.id).scope == in_mem_dn.scope          assert _DataManager._get(in_mem_dn.id).owner_id == "Scenario_id"          assert _DataManager._get(in_mem_dn.id).owner_id == in_mem_dn.owner_id          assert _DataManager._get(in_mem_dn.id).parent_ids == {"task_id"}          assert _DataManager._get(in_mem_dn.id).parent_ids == in_mem_dn.parent_ids          assert _DataManager._get(in_mem_dn.id).last_edit_date is not None          assert _DataManager._get(in_mem_dn.id).last_edit_date == in_mem_dn.last_edit_date          assert _DataManager._get(in_mem_dn.id).job_ids == []          assert _DataManager._get(in_mem_dn.id).job_ids == in_mem_dn.job_ids          assert _DataManager._get(in_mem_dn.id).is_ready_for_reading          assert _DataManager._get(in_mem_dn.id).is_ready_for_reading == in_mem_dn.is_ready_for_reading          assert len(_DataManager._get(in_mem_dn.id).properties) == 1          assert _DataManager._get(in_mem_dn.id).properties.get("other_data") == "foo"          assert _DataManager._get(in_mem_dn.id).properties == in_mem_dn.properties            assert _DataManager._get(in_mem_dn) is not None          assert _DataManager._get(in_mem_dn).id == in_mem_dn.id          assert _DataManager._get(in_mem_dn).config_id == "baz"          assert _DataManager._get(in_mem_dn).config_id == in_mem_dn.config_id          assert _DataManager._get(in_mem_dn).scope == Scope.SCENARIO          assert _DataManager._get(in_mem_dn).scope == in_mem_dn.scope          assert _DataManager._get(in_mem_dn).owner_id == "Scenario_id"          assert _DataManager._get(in_mem_dn).owner_id == in_mem_dn.owner_id          assert _DataManager._get(in_mem_dn).parent_ids == {"task_id"}          assert _DataManager._get(in_mem_dn).parent_ids == in_mem_dn.parent_ids          assert _DataManager._get(in_mem_dn).last_edit_date is not None          assert _DataManager._get(in_mem_dn).last_edit_date == in_mem_dn.last_edit_date          assert _DataManager._get(in_mem_dn).job_ids == []          assert _DataManager._get(in_mem_dn).job_ids == in_mem_dn.job_ids          assert _DataManager._get(in_mem_dn).is_ready_for_reading          assert _DataManager._get(in_mem_dn).is_ready_for_reading == in_mem_dn.is_ready_for_reading          assert len(_DataManager._get(in_mem_dn).properties) == 1          assert _DataManager._get(in_mem_dn).properties.get("other_data") == "foo"          assert _DataManager._get(in_mem_dn).properties == in_mem_dn.properties        def test_create_and_get_pickle_data_node(self):          # Test we can instantiate a PickleDataNode from DataNodeConfig with :          # - an in_memory type          # - a business cycle scope          # - No owner id          # - no default data          dn_config = Config.configure_data_node(id="plop", storage_type="pickle", scope=Scope.CYCLE)          pickle_dn = _DataManager._create_and_set(dn_config, None, {"task_id_1", "task_id_2"})            assert isinstance(pickle_dn, PickleDataNode)          assert isinstance(_DataManager._get(pickle_dn.id), PickleDataNode)            assert _DataManager._exists(pickle_dn.id)          assert _DataManager._get(pickle_dn.id) is not None          assert _DataManager._get(pickle_dn.id).id == pickle_dn.id          assert _DataManager._get(pickle_dn.id).config_id == "plop"          assert _DataManager._get(pickle_dn.id).config_id == pickle_dn.config_id          assert _DataManager._get(pickle_dn.id).scope == Scope.CYCLE          assert _DataManager._get(pickle_dn.id).scope == pickle_dn.scope          assert _DataManager._get(pickle_dn.id).owner_id is None          assert _DataManager._get(pickle_dn.id).owner_id == pickle_dn.owner_id          assert _DataManager._get(pickle_dn.id).parent_ids == {"task_id_1", "task_id_2"}          assert _DataManager._get(pickle_dn.id).parent_ids == pickle_dn.parent_ids          assert _DataManager._get(pickle_dn.id).last_edit_date is None          assert _DataManager._get(pickle_dn.id).last_edit_date == pickle_dn.last_edit_date          assert _DataManager._get(pickle_dn.id).job_ids == []          assert _DataManager._get(pickle_dn.id).job_ids == pickle_dn.job_ids          assert not _DataManager._get(pickle_dn.id).is_ready_for_reading          assert _DataManager._get(pickle_dn.id).is_ready_for_reading == pickle_dn.is_ready_for_reading          assert len(_DataManager._get(pickle_dn.id).properties) == 1          assert _DataManager._get(pickle_dn.id).properties == pickle_dn.properties            assert _DataManager._get(pickle_dn) is not None          assert _DataManager._get(pickle_dn).id == pickle_dn.id          assert _DataManager._get(pickle_dn).config_id == "plop"          assert _DataManager._get(pickle_dn).config_id == pickle_dn.config_id          assert _DataManager._get(pickle_dn).scope == Scope.CYCLE          assert _DataManager._get(pickle_dn).scope == pickle_dn.scope          assert _DataManager._get(pickle_dn).owner_id is None          assert _DataManager._get(pickle_dn).owner_id == pickle_dn.owner_id          assert _DataManager._get(pickle_dn).parent_ids == {"task_id_1", "task_id_2"}          assert _DataManager._get(pickle_dn).parent_ids == pickle_dn.parent_ids          assert _DataManager._get(pickle_dn).last_edit_date is None          assert _DataManager._get(pickle_dn).last_edit_date == pickle_dn.last_edit_date          assert _DataManager._get(pickle_dn).job_ids == []          assert _DataManager._get(pickle_dn).job_ids == pickle_dn.job_ids          assert not _DataManager._get(pickle_dn).is_ready_for_reading          assert _DataManager._get(pickle_dn).is_ready_for_reading == pickle_dn.is_ready_for_reading          assert len(_DataManager._get(pickle_dn).properties) == 1          assert _DataManager._get(pickle_dn).properties == pickle_dn.properties        def test_create_raises_exception_with_wrong_type(self):          wrong_type_dn_config = DataNodeConfig(id="foo", storage_type="bar", scope=DataNodeConfig._DEFAULT_SCOPE)          with pytest.raises(InvalidDataNodeType):              _DataManager._create_and_set(wrong_type_dn_config, None, None)        def test_create_from_same_config_generates_new_data_node_and_new_id(self):          dn_config = Config.configure_data_node(id="foo", storage_type="in_memory")          dn = _DataManager._create_and_set(dn_config, None, None)          dn_2 = _DataManager._create_and_set(dn_config, None, None)          assert dn_2.id != dn.id        def test_create_uses_overridden_attributes_in_config_file(self):          Config.override(os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/config.toml"))            csv_dn_cfg = Config.configure_data_node(id="foo", storage_type="csv", path="bar", has_header=True)          csv_dn = _DataManager._create_and_set(csv_dn_cfg, None, None)          assert csv_dn.config_id == "foo"          assert isinstance(csv_dn, CSVDataNode)          assert csv_dn._path == "path_from_config_file"          assert csv_dn.has_header            csv_dn_cfg = Config.configure_data_node(id="baz", storage_type="csv", path="bar", has_header=True)          csv_dn = _DataManager._create_and_set(csv_dn_cfg, None, None)          assert csv_dn.config_id == "baz"          assert isinstance(csv_dn, CSVDataNode)          assert csv_dn._path == "bar"          assert csv_dn.has_header        def test_get_if_not_exists(self):          with pytest.raises(ModelNotFound):              _DataManager._repository._load("test_data_node_2")        def test_get_all(self):          assert len(_DataManager._get_all()) == 0          dn_config_1 = Config.configure_data_node(id="foo", storage_type="in_memory")          _DataManager._create_and_set(dn_config_1, None, None)          assert len(_DataManager._get_all()) == 1          dn_config_2 = Config.configure_data_node(id="baz", storage_type="in_memory")          _DataManager._create_and_set(dn_config_2, None, None)          _DataManager._create_and_set(dn_config_2, None, None)          assert len(_DataManager._get_all()) == 3          assert len([dn for dn in _DataManager._get_all() if dn.config_id == "foo"]) == 1          assert len([dn for dn in _DataManager._get_all() if dn.config_id == "baz"]) == 2        def test_get_all_on_multiple_versions_environment(self):          # Create 5 data nodes with 2 versions each          # Only version 1.0 has the data node with config_id = "config_id_1"          # Only version 2.0 has the data node with config_id = "config_id_6"          for version in range(1, 3):              for i in range(5):                  _DataManager._set(                      InMemoryDataNode(                          f"config_id_{i + version}",                          Scope.SCENARIO,                          id=DataNodeId(f"id{i}_v{version}"),                          version=f"{version}.0",                      )                  )            _VersionManager._set_experiment_version("1.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_1"}])) == 1          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_6"}])) == 0            _VersionManager._set_development_version("1.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_1"}])) == 1          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_6"}])) == 0            _VersionManager._set_experiment_version("2.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_1"}])) == 0          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_6"}])) == 1            _VersionManager._set_development_version("2.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_1"}])) == 0          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_6"}])) == 1        def test_set(self):          dn = InMemoryDataNode(              "config_id",              Scope.SCENARIO,              id=DataNodeId("id"),              owner_id=None,              parent_ids={"task_id_1"},              last_edit_date=None,              edits=[],              edit_in_progress=False,              properties={"foo": "bar"},          )          assert len(_DataManager._get_all()) == 0          assert not _DataManager._exists(dn.id)          _DataManager._set(dn)          assert len(_DataManager._get_all()) == 1          assert _DataManager._exists(dn.id)            # changing data node attribute          dn.config_id = "foo"          assert dn.config_id == "foo"          _DataManager._set(dn)          assert len(_DataManager._get_all()) == 1          assert dn.config_id == "foo"          assert _DataManager._get(dn.id).config_id == "foo"        def test_delete(self):          dn_1 = InMemoryDataNode("config_id", Scope.SCENARIO, id="id_1")          dn_2 = InMemoryDataNode("config_id", Scope.SCENARIO, id="id_2")          dn_3 = InMemoryDataNode("config_id", Scope.SCENARIO, id="id_3")          assert len(_DataManager._get_all()) == 0          _DataManager._set(dn_1)          _DataManager._set(dn_2)          _DataManager._set(dn_3)          assert len(_DataManager._get_all()) == 3          assert all(_DataManager._exists(dn.id) for dn in [dn_1, dn_2, dn_3])          _DataManager._delete(dn_1.id)          assert len(_DataManager._get_all()) == 2          assert _DataManager._get(dn_2.id).id == dn_2.id          assert _DataManager._get(dn_3.id).id == dn_3.id          assert _DataManager._get(dn_1.id) is None          assert all(_DataManager._exists(dn.id) for dn in [dn_2, dn_3])          assert not _DataManager._exists(dn_1.id)          _DataManager._delete_all()          assert len(_DataManager._get_all()) == 0          assert not any(_DataManager._exists(dn.id) for dn in [dn_2, dn_3])        def test_get_or_create(self):          def _get_or_create_dn(config, *args):              return _DataManager._bulk_get_or_create([config], *args)[config]            _DataManager._delete_all()            global_dn_config = Config.configure_data_node(              id="test_data_node", storage_type="in_memory", scope=Scope.GLOBAL, data="In memory Data Node"          )          cycle_dn_config = Config.configure_data_node(              id="test_data_node1", storage_type="in_memory", scope=Scope.CYCLE, data="In memory Data Node"          )          scenario_dn_config = Config.configure_data_node(              id="test_data_node2", storage_type="in_memory", scope=Scope.SCENARIO, data="In memory scenario"          )            assert len(_DataManager._get_all()) == 0          global_dn = _get_or_create_dn(global_dn_config, None, None)          assert len(_DataManager._get_all()) == 1          global_dn_bis = _get_or_create_dn(global_dn_config, None)          assert len(_DataManager._get_all()) == 1          assert global_dn.id == global_dn_bis.id            scenario_dn = _get_or_create_dn(scenario_dn_config, None, "scenario_id")          assert len(_DataManager._get_all()) == 2          scenario_dn_bis = _get_or_create_dn(scenario_dn_config, None, "scenario_id")          assert len(_DataManager._get_all()) == 2          assert scenario_dn.id == scenario_dn_bis.id          scenario_dn_ter = _get_or_create_dn(scenario_dn_config, None, "scenario_id")          assert len(_DataManager._get_all()) == 2          assert scenario_dn.id == scenario_dn_bis.id          assert scenario_dn_bis.id == scenario_dn_ter.id          scenario_dn_quater = _get_or_create_dn(scenario_dn_config, None, "scenario_id_2")          assert len(_DataManager._get_all()) == 3          assert scenario_dn.id == scenario_dn_bis.id          assert scenario_dn_bis.id == scenario_dn_ter.id          assert scenario_dn_ter.id != scenario_dn_quater.id            assert len(_DataManager._get_all()) == 3          cycle_dn = _get_or_create_dn(cycle_dn_config, "cycle_id", None)          assert len(_DataManager._get_all()) == 4          cycle_dn_1 = _get_or_create_dn(cycle_dn_config, "cycle_id", None)          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_1.id          cycle_dn_2 = _get_or_create_dn(cycle_dn_config, "cycle_id", "scenario_id")          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_2.id          cycle_dn_3 = _get_or_create_dn(cycle_dn_config, "cycle_id", None)          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_3.id          cycle_dn_4 = _get_or_create_dn(cycle_dn_config, "cycle_id", "scenario_id")          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_4.id          cycle_dn_5 = _get_or_create_dn(cycle_dn_config, "cycle_id", "scenario_id_2")          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_5.id            assert cycle_dn_1.id == cycle_dn_2.id          assert cycle_dn_2.id == cycle_dn_3.id          assert cycle_dn_3.id == cycle_dn_4.id          assert cycle_dn_4.id == cycle_dn_5.id        def test_ensure_persistence_of_data_node(self):          dm = _DataManager()          dm._delete_all()            dn_config_1 = Config.configure_data_node(              id="data_node_1", storage_type="in_memory", data="In memory sequence 2"          )          dn_config_2 = Config.configure_data_node(              id="data_node_2", storage_type="in_memory", data="In memory sequence 2"          )            dm._bulk_get_or_create([dn_config_1, dn_config_2])            assert len(dm._get_all()) == 2            # Delete the DataManager to ensure it's get from the storage system          del dm          dm = _DataManager()          dm._bulk_get_or_create([dn_config_1])          assert len(dm._get_all()) == 2            dm._delete_all()        def test_clean_generated_pickle_files(self, pickle_file_path):          user_pickle_dn_config = Config.configure_data_node(              id="d1", storage_type="pickle", path=pickle_file_path, default_data="d"          )          generated_pickle_dn_1_config = Config.configure_data_node(id="d2", storage_type="pickle", default_data="d")          generated_pickle_dn_2_config = Config.configure_data_node(id="d3", storage_type="pickle", default_data="d")            dns = _DataManager._bulk_get_or_create(              [user_pickle_dn_config, generated_pickle_dn_1_config, generated_pickle_dn_2_config]          )            user_pickle_dn = dns[user_pickle_dn_config]          generated_pickle_dn_1 = dns[generated_pickle_dn_1_config]          generated_pickle_dn_2 = dns[generated_pickle_dn_2_config]            _DataManager._clean_pickle_file(user_pickle_dn.id)          assert file_exists(user_pickle_dn.path)            _DataManager._clean_pickle_files([generated_pickle_dn_1, generated_pickle_dn_2])          assert not file_exists(generated_pickle_dn_1.path)          assert not file_exists(generated_pickle_dn_2.path)        def test_delete_does_clean_generated_pickle_files(self, pickle_file_path):          user_pickle_dn_config = Config.configure_data_node(              id="d1", storage_type="pickle", path=pickle_file_path, default_data="d"          )          generated_pickle_dn_config_1 = Config.configure_data_node(id="d2", storage_type="pickle", default_data="d")          generated_pickle_dn_config_2 = Config.configure_data_node(id="d3", storage_type="pickle", default_data="d")          generated_pickle_dn_config_3 = Config.configure_data_node(id="d4", storage_type="pickle", default_data="d")            dns = _DataManager._bulk_get_or_create(              [                  user_pickle_dn_config,                  generated_pickle_dn_config_1,                  generated_pickle_dn_config_2,                  generated_pickle_dn_config_3,              ]          )            user_pickle_dn = dns[user_pickle_dn_config]          generated_pickle_dn_1 = dns[generated_pickle_dn_config_1]          generated_pickle_dn_2 = dns[generated_pickle_dn_config_2]          generated_pickle_dn_3 = dns[generated_pickle_dn_config_3]            _DataManager._delete(user_pickle_dn.id)          assert file_exists(user_pickle_dn.path)            _DataManager._delete_many([generated_pickle_dn_1.id, generated_pickle_dn_2.id])          assert not file_exists(generated_pickle_dn_1.path)          assert not file_exists(generated_pickle_dn_2.path)            _DataManager._delete_all()          assert not file_exists(generated_pickle_dn_3.path)        def test_create_dn_from_loaded_config_no_scope(self):          file_config = NamedTemporaryFile(              """              [TAIPY]                [DATA_NODE.a]              default_data = "4:int"                [DATA_NODE.b]                [TASK.t]              function = "math.sqrt:function"              inputs = [ "a:SECTION",]              outputs = [ "b:SECTION",]              skippable = "False:bool"                [SCENARIO.s]              tasks = [ "t:SECTION",]              sequences.s_sequence = [ "t:SECTION",]                [SCENARIO.s.comparators]              """          )          from src.taipy import core as tp            Config.override(file_config.filename)          tp.create_scenario(Config.scenarios["s"])          tp.create_scenario(Config.scenarios["s"])            assert len(tp.get_data_nodes()) == 4        def test_create_dn_from_loaded_config_no_storage_type(self):          file_config = NamedTemporaryFile(              """              [TAIPY]                [DATA_NODE.input]              scope = "SCENARIO:SCOPE"              default_data = "21:int"                [DATA_NODE.output]              storage_type = "in_memory"              scope = "SCENARIO:SCOPE"                [TASK.double]              inputs = [ "input:SECTION",]              function = "math.sqrt:function"              outputs = [ "output:SECTION",]              skippable = "False:bool"                [SCENARIO.my_scenario]              tasks = [ "double:SECTION",]              sequences.my_sequence = [ "double:SECTION",]                [SCENARIO.my_scenario.comparators]              """          )          from src.taipy import core as tp            Config.override(file_config.filename)          scenario = tp.create_scenario(Config.scenarios["my_scenario"])            assert isinstance(scenario.input, PickleDataNode)          assert isinstance(scenario.output, InMemoryDataNode)        def test_create_dn_from_loaded_config_modified_default_config(self):          file_config = NamedTemporaryFile(              """              [TAIPY]                [DATA_NODE.input]              scope = "SCENARIO:SCOPE"              default_path="fake/path.csv"                [DATA_NODE.output]              storage_type = "in_memory"              scope = "SCENARIO:SCOPE"                [TASK.double]              inputs = [ "input:SECTION",]              function = "math.sqrt:function"              outputs = [ "output:SECTION",]              skippable = "False:bool"                [SCENARIO.my_scenario]              tasks = [ "double:SECTION",]              sequences.my_sequence = [ "double:SECTION",]                [SCENARIO.my_scenario.comparators]              """          )          from src.taipy import core as tp            Config.set_default_data_node_configuration(storage_type="csv")          Config.override(file_config.filename)          scenario = tp.create_scenario(Config.scenarios["my_scenario"])            assert isinstance(scenario.input, CSVDataNode)          assert isinstance(scenario.output, InMemoryDataNode)        def test_get_tasks_by_config_id(self):          dn_config_1 = Config.configure_data_node("dn_1", scope=Scope.SCENARIO)          dn_config_2 = Config.configure_data_node("dn_2", scope=Scope.SCENARIO)          dn_config_3 = Config.configure_data_node("dn_3", scope=Scope.SCENARIO)            dn_1_1 = _DataManager._create_and_set(dn_config_1, None, None)          dn_1_2 = _DataManager._create_and_set(dn_config_1, None, None)          dn_1_3 = _DataManager._create_and_set(dn_config_1, None, None)          assert len(_DataManager._get_all()) == 3            dn_2_1 = _DataManager._create_and_set(dn_config_2, None, None)          dn_2_2 = _DataManager._create_and_set(dn_config_2, None, None)          assert len(_DataManager._get_all()) == 5            dn_3_1 = _DataManager._create_and_set(dn_config_3, None, None)          assert len(_DataManager._get_all()) == 6            dn_1_datanodes = _DataManager._get_by_config_id(dn_config_1.id)          assert len(dn_1_datanodes) == 3          assert sorted([dn_1_1.id, dn_1_2.id, dn_1_3.id]) == sorted([sequence.id for sequence in dn_1_datanodes])            dn_2_datanodes = _DataManager._get_by_config_id(dn_config_2.id)          assert len(dn_2_datanodes) == 2          assert sorted([dn_2_1.id, dn_2_2.id]) == sorted([sequence.id for sequence in dn_2_datanodes])            dn_3_datanodes = _DataManager._get_by_config_id(dn_config_3.id)          assert len(dn_3_datanodes) == 1          assert sorted([dn_3_1.id]) == sorted([sequence.id for sequence in dn_3_datanodes])        def test_get_data_nodes_by_config_id_in_multiple_versions_environment(self):          dn_config_1 = Config.configure_data_node("dn_1", scope=Scope.SCENARIO)          dn_config_2 = Config.configure_data_node("dn_2", scope=Scope.SCENARIO)            _VersionManager._set_experiment_version("1.0")          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_2, None, None)          _DataManager._create_and_set(dn_config_2, None, None)            assert len(_DataManager._get_by_config_id(dn_config_1.id)) == 3          assert len(_DataManager._get_by_config_id(dn_config_2.id)) == 2            _VersionManager._set_experiment_version("2.0")          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_2, None, None)          _DataManager._create_and_set(dn_config_2, None, None)            assert len(_DataManager._get_by_config_id(dn_config_1.id)) == 3          assert len(_DataManager._get_by_config_id(dn_config_2.id)) == 2


from typing import Dict, List    import numpy as np  import pandas as pd  import pytest    from src.taipy.core.data.operator import JoinOperator, Operator    from .utils import (      CustomClass,      FakeCustomDataNode,      FakeDataframeDataNode,      FakeDataNode,      FakeListDataNode,      FakeMultiSheetExcelCustomDataNode,      FakeMultiSheetExcelDataFrameDataNode,      FakeNumpyarrayDataNode,  )      def test_filter_pandas_exposed_type(default_data_frame):      dn = FakeDataNode("fake_dn")      dn.write("Any data")        with pytest.raises(NotImplementedError):          dn.filter((("any", 0, Operator.EQUAL)), JoinOperator.OR)      with pytest.raises(NotImplementedError):          dn.filter((("any", 0, Operator.NOT_EQUAL)), JoinOperator.OR)      with pytest.raises(NotImplementedError):          dn.filter((("any", 0, Operator.LESS_THAN)), JoinOperator.AND)      with pytest.raises(NotImplementedError):          dn.filter((("any", 0, Operator.LESS_OR_EQUAL)), JoinOperator.AND)      with pytest.raises(NotImplementedError):          dn.filter((("any", 0, Operator.GREATER_THAN)))      with pytest.raises(NotImplementedError):          dn.filter(("any", 0, Operator.GREATER_OR_EQUAL))        df_dn = FakeDataframeDataNode("fake_dataframe_dn", default_data_frame)        COLUMN_NAME_1 = "a"      COLUMN_NAME_2 = "b"      assert len(df_dn.filter((COLUMN_NAME_1, 1, Operator.EQUAL))) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] == 1]      )      assert len(df_dn.filter((COLUMN_NAME_1, 1, Operator.NOT_EQUAL))) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] != 1]      )      assert len(df_dn.filter([(COLUMN_NAME_1, 1, Operator.EQUAL)])) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] == 1]      )      assert len(df_dn.filter([(COLUMN_NAME_1, 1, Operator.NOT_EQUAL)])) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] != 1]      )      assert len(df_dn.filter([(COLUMN_NAME_1, 1, Operator.LESS_THAN)])) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] < 1]      )      assert len(df_dn.filter([(COLUMN_NAME_1, 1, Operator.LESS_OR_EQUAL)])) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] <= 1]      )      assert len(df_dn.filter([(COLUMN_NAME_1, 1, Operator.GREATER_THAN)])) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] > 1]      )      assert len(df_dn.filter([(COLUMN_NAME_1, 1, Operator.GREATER_OR_EQUAL)])) == len(          default_data_frame[default_data_frame[COLUMN_NAME_1] >= 1]      )      assert len(df_dn.filter([(COLUMN_NAME_1, -1000, Operator.LESS_OR_EQUAL)])) == 0      assert len(df_dn.filter([(COLUMN_NAME_1, 1000, Operator.GREATER_OR_EQUAL)])) == 0      assert len(df_dn.filter([(COLUMN_NAME_1, 4, Operator.EQUAL), (COLUMN_NAME_1, 5, Operator.EQUAL)])) == len(          default_data_frame[(default_data_frame[COLUMN_NAME_1] == 4) & (default_data_frame[COLUMN_NAME_1] == 5)]      )      assert len(          df_dn.filter([(COLUMN_NAME_1, 4, Operator.EQUAL), (COLUMN_NAME_2, 5, Operator.EQUAL)], JoinOperator.OR)      ) == len(default_data_frame[(default_data_frame[COLUMN_NAME_1] == 4) | (default_data_frame[COLUMN_NAME_2] == 5)])      assert len(          df_dn.filter(              [(COLUMN_NAME_1, 1, Operator.GREATER_THAN), (COLUMN_NAME_2, 3, Operator.GREATER_THAN)], JoinOperator.AND          )      ) == len(default_data_frame[(default_data_frame[COLUMN_NAME_1] > 1) & (default_data_frame[COLUMN_NAME_2] > 3)])      assert len(          df_dn.filter(              [(COLUMN_NAME_1, 2, Operator.GREATER_THAN), (COLUMN_NAME_1, 3, Operator.GREATER_THAN)], JoinOperator.OR          )      ) == len(default_data_frame[(default_data_frame[COLUMN_NAME_1] > 2) | (default_data_frame[COLUMN_NAME_1] > 3)])      assert len(          df_dn.filter(              [(COLUMN_NAME_1, 10, Operator.GREATER_THAN), (COLUMN_NAME_1, -10, Operator.LESS_THAN)], JoinOperator.AND          )      ) == len(default_data_frame[(default_data_frame[COLUMN_NAME_1] > 10) | (default_data_frame[COLUMN_NAME_1] < -10)])      assert len(          df_dn.filter(              [(COLUMN_NAME_1, 10, Operator.GREATER_THAN), (COLUMN_NAME_1, -10, Operator.LESS_THAN)], JoinOperator.OR          )      ) == len(default_data_frame[(default_data_frame[COLUMN_NAME_1] > 10) | (default_data_frame[COLUMN_NAME_1] < -10)])      def test_filter_list():      list_dn = FakeListDataNode("fake_list_dn")        KEY_NAME = "value"        assert len(list_dn.filter((KEY_NAME, 4, Operator.EQUAL))) == 1      assert len(list_dn.filter((KEY_NAME, 4, Operator.NOT_EQUAL))) == 9      assert len(list_dn.filter([(KEY_NAME, 4, Operator.EQUAL)])) == 1      assert len(list_dn.filter([(KEY_NAME, 4, Operator.NOT_EQUAL)])) == 9      assert len(list_dn.filter([(KEY_NAME, 4, Operator.LESS_THAN)])) == 4      assert len(list_dn.filter([(KEY_NAME, 4, Operator.LESS_OR_EQUAL)])) == 5      assert len(list_dn.filter([(KEY_NAME, 4, Operator.GREATER_THAN)])) == 5      assert len(list_dn.filter([(KEY_NAME, 4, Operator.GREATER_OR_EQUAL)])) == 6      assert len(list_dn.filter([(KEY_NAME, -1000, Operator.LESS_OR_EQUAL)])) == 0      assert len(list_dn.filter([(KEY_NAME, 1000, Operator.GREATER_OR_EQUAL)])) == 0        assert len(list_dn.filter([(KEY_NAME, 4, Operator.EQUAL), (KEY_NAME, 5, Operator.EQUAL)])) == 0      assert len(list_dn.filter([(KEY_NAME, 4, Operator.EQUAL), (KEY_NAME, 5, Operator.EQUAL)], JoinOperator.OR)) == 2      assert len(list_dn.filter([(KEY_NAME, 4, Operator.EQUAL), (KEY_NAME, 11, Operator.EQUAL)], JoinOperator.AND)) == 0      assert len(list_dn.filter([(KEY_NAME, 4, Operator.EQUAL), (KEY_NAME, 11, Operator.EQUAL)], JoinOperator.OR)) == 1        assert (          len(list_dn.filter([(KEY_NAME, -10, Operator.LESS_OR_EQUAL), (KEY_NAME, 11, Operator.GREATER_OR_EQUAL)])) == 0      )      assert (          len(              list_dn.filter(                  [                      (KEY_NAME, 4, Operator.GREATER_OR_EQUAL),                      (KEY_NAME, 6, Operator.GREATER_OR_EQUAL),                  ],                  JoinOperator.AND,              )          )          == 4      )      assert (          len(              list_dn.filter(                  [                      (KEY_NAME, 4, Operator.GREATER_OR_EQUAL),                      (KEY_NAME, 6, Operator.GREATER_OR_EQUAL),                      (KEY_NAME, 11, Operator.EQUAL),                  ],                  JoinOperator.AND,              )          )          == 0      )      assert (          len(              list_dn.filter(                  [                      (KEY_NAME, 4, Operator.GREATER_OR_EQUAL),                      (KEY_NAME, 6, Operator.GREATER_OR_EQUAL),                      (KEY_NAME, 11, Operator.EQUAL),                  ],                  JoinOperator.OR,              )          )          == 6      )      def test_filter_numpy_exposed_type(default_data_frame):      default_array = default_data_frame.to_numpy()        df_dn = FakeNumpyarrayDataNode("fake_dataframe_dn", default_array)        assert len(df_dn.filter((0, 1, Operator.EQUAL))) == len(default_array[default_array[:, 0] == 1])      assert len(df_dn.filter((0, 1, Operator.NOT_EQUAL))) == len(default_array[default_array[:, 0] != 1])      assert len(df_dn.filter([(0, 1, Operator.EQUAL)])) == len(default_array[default_array[:, 0] == 1])      assert len(df_dn.filter([(0, 1, Operator.NOT_EQUAL)])) == len(default_array[default_array[:, 0] != 1])      assert len(df_dn.filter([(0, 1, Operator.LESS_THAN)])) == len(default_array[default_array[:, 0] < 1])      assert len(df_dn.filter([(0, 1, Operator.LESS_OR_EQUAL)])) == len(default_array[default_array[:, 0] <= 1])      assert len(df_dn.filter([(0, 1, Operator.GREATER_THAN)])) == len(default_array[default_array[:, 0] > 1])      assert len(df_dn.filter([(0, 1, Operator.GREATER_OR_EQUAL)])) == len(default_array[default_array[:, 0] >= 1])      assert len(df_dn.filter([(0, -1000, Operator.LESS_OR_EQUAL)])) == 0      assert len(df_dn.filter([(0, 1000, Operator.GREATER_OR_EQUAL)])) == 0      assert len(df_dn.filter([(0, 4, Operator.EQUAL), (0, 5, Operator.EQUAL)])) == len(          default_array[(default_array[:, 0] == 4) & (default_array[:, 0] == 5)]      )      assert len(df_dn.filter([(0, 4, Operator.EQUAL), (1, 5, Operator.EQUAL)], JoinOperator.OR)) == len(          default_array[(default_array[:, 0] == 4) | (default_array[:, 1] == 5)]      )      assert len(df_dn.filter([(0, 1, Operator.GREATER_THAN), (1, 3, Operator.GREATER_THAN)], JoinOperator.AND)) == len(          default_array[(default_array[:, 0] > 1) & (default_array[:, 1] > 3)]      )      assert len(df_dn.filter([(0, 2, Operator.GREATER_THAN), (0, 3, Operator.GREATER_THAN)], JoinOperator.OR)) == len(          default_array[(default_array[:, 0] > 2) | (default_array[:, 0] > 3)]      )      assert len(df_dn.filter([(0, 10, Operator.GREATER_THAN), (0, -10, Operator.LESS_THAN)], JoinOperator.AND)) == len(          default_array[(default_array[:, 0] > 10) | (default_array[:, 0] < -10)]      )      assert len(df_dn.filter([(0, 10, Operator.GREATER_THAN), (0, -10, Operator.LESS_THAN)], JoinOperator.OR)) == len(          default_array[(default_array[:, 0] > 10) | (default_array[:, 0] < -10)]      )      def test_filter_by_get_item(default_data_frame):      # get item for DataFrame data_type      default_data_frame[1] = [100, 100]      df_dn = FakeDataframeDataNode("fake_dataframe_dn", default_data_frame)        filtered_df_dn = df_dn["a"]      assert isinstance(filtered_df_dn, pd.Series)      assert len(filtered_df_dn) == len(default_data_frame["a"])      assert filtered_df_dn.to_dict() == default_data_frame["a"].to_dict()        filtered_df_dn = df_dn[1]      assert isinstance(filtered_df_dn, pd.Series)      assert len(filtered_df_dn) == len(default_data_frame[1])      assert filtered_df_dn.to_dict() == default_data_frame[1].to_dict()        filtered_df_dn = df_dn[0:2]      assert isinstance(filtered_df_dn, pd.DataFrame)      assert filtered_df_dn.shape == default_data_frame[0:2].shape      assert len(filtered_df_dn) == 2        bool_df = default_data_frame.copy(deep=True) > 4      filtered_df_dn = df_dn[bool_df]      assert isinstance(filtered_df_dn, pd.DataFrame)        bool_1d_index = [True, False]      filtered_df_dn = df_dn[bool_1d_index]      assert isinstance(filtered_df_dn, pd.DataFrame)      assert filtered_df_dn.to_dict() == default_data_frame[bool_1d_index].to_dict()      assert len(filtered_df_dn) == 1        filtered_df_dn = df_dn[["a", "b"]]      assert isinstance(filtered_df_dn, pd.DataFrame)      assert filtered_df_dn.shape == default_data_frame[["a", "b"]].shape      assert filtered_df_dn.to_dict() == default_data_frame[["a", "b"]].to_dict()        # get item for custom data_type      custom_dn = FakeCustomDataNode("fake_custom_dn")        filtered_custom_dn = custom_dn["a"]      assert isinstance(filtered_custom_dn, List)      assert len(filtered_custom_dn) == 10      assert filtered_custom_dn == [i for i in range(10)]        filtered_custom_dn = custom_dn[0:5]      assert isinstance(filtered_custom_dn, List)      assert all([isinstance(x, CustomClass) for x in filtered_custom_dn])      assert len(filtered_custom_dn) == 5        bool_1d_index = [True if i < 5 else False for i in range(10)]      filtered_custom_dn = custom_dn[bool_1d_index]      assert isinstance(filtered_custom_dn, List)      assert len(filtered_custom_dn) == 5      assert filtered_custom_dn == custom_dn._read()[:5]        filtered_custom_dn = custom_dn[["a", "b"]]      assert isinstance(filtered_custom_dn, List)      assert all([isinstance(x, Dict) for x in filtered_custom_dn])      assert len(filtered_custom_dn) == 10      assert filtered_custom_dn == [{"a": i, "b": i * 2} for i in range(10)]        # get item for Multi-sheet Excel data_type      multi_sheet_excel_df_dn = FakeMultiSheetExcelDataFrameDataNode("fake_multi_sheet_excel_df_dn", default_data_frame)      filtered_multi_sheet_excel_df_dn = multi_sheet_excel_df_dn["Sheet1"]      assert isinstance(filtered_multi_sheet_excel_df_dn, pd.DataFrame)      assert len(filtered_multi_sheet_excel_df_dn) == len(default_data_frame)      assert np.array_equal(filtered_multi_sheet_excel_df_dn.to_numpy(), default_data_frame.to_numpy())        multi_sheet_excel_custom_dn = FakeMultiSheetExcelCustomDataNode("fake_multi_sheet_excel_df_dn")      filtered_multi_sheet_excel_custom_dn = multi_sheet_excel_custom_dn["Sheet1"]      assert isinstance(filtered_multi_sheet_excel_custom_dn, List)      assert len(filtered_multi_sheet_excel_custom_dn) == 10      expected_value = [CustomClass(i, i * 2) for i in range(10)]      assert all(          [              expected.a == filtered.a and expected.b == filtered.b              for expected, filtered in zip(expected_value, filtered_multi_sheet_excel_custom_dn)          ]      )


import os  import pathlib  from datetime import datetime  from importlib import util  from time import sleep    import modin.pandas as modin_pd  import numpy as np  import pandas as pd  import pytest  from modin.pandas.test.utils import df_equals  from pandas.testing import assert_frame_equal    from src.taipy.core.data._data_manager import _DataManager  from src.taipy.core.data.data_node_id import DataNodeId  from src.taipy.core.data.operator import JoinOperator, Operator  from src.taipy.core.data.parquet import ParquetDataNode  from src.taipy.core.exceptions.exceptions import (      InvalidExposedType,      NoData,      UnknownCompressionAlgorithm,      UnknownParquetEngine,  )  from taipy.config.common.scope import Scope  from taipy.config.config import Config  from taipy.config.exceptions.exceptions import InvalidConfigurationId      @pytest.fixture(scope="function", autouse=True)  def cleanup():      yield      path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/temp.parquet")      if os.path.isfile(path):          os.remove(path)      class MyCustomObject:      def __init__(self, id, integer, text):          self.id = id          self.integer = integer          self.text = text      class MyOtherCustomObject:      def __init__(self, id, sentence):          self.id = id          self.sentence = sentence      def create_custom_class(**kwargs):      return MyOtherCustomObject(id=kwargs["id"], sentence=kwargs["text"])      class TestParquetDataNode:      __engine = ["pyarrow"]      if util.find_spec("fastparquet"):          __engine.append("fastparquet")        def test_create(self):          path = "data/node/path"          compression = "snappy"          dn = ParquetDataNode(              "foo_bar", Scope.SCENARIO, properties={"path": path, "compression": compression, "name": "super name"}          )          assert isinstance(dn, ParquetDataNode)          assert dn.storage_type() == "parquet"          assert dn.config_id == "foo_bar"          assert dn.name == "super name"          assert dn.scope == Scope.SCENARIO          assert dn.id is not None          assert dn.owner_id is None          assert dn.last_edit_date is None          assert dn.job_ids == []          assert not dn.is_ready_for_reading          assert dn.path == path          assert dn.exposed_type == "pandas"          assert dn.compression == "snappy"          assert dn.engine == "pyarrow"            with pytest.raises(InvalidConfigurationId):              dn = ParquetDataNode("foo bar", Scope.SCENARIO, properties={"path": path, "name": "super name"})        def test_get_user_properties(self, parquet_file_path):          dn_1 = ParquetDataNode("dn_1", Scope.SCENARIO, properties={"path": parquet_file_path})          assert dn_1._get_user_properties() == {}            dn_2 = ParquetDataNode(              "dn_2",              Scope.SCENARIO,              properties={                  "exposed_type": "numpy",                  "default_data": "foo",                  "default_path": parquet_file_path,                  "engine": "pyarrow",                  "compression": "snappy",                  "read_kwargs": {"columns": ["a", "b"]},                  "write_kwargs": {"index": False},                  "foo": "bar",              },          )            # exposed_type, default_data, default_path, path, engine, compression, read_kwargs, write_kwargs          # are filtered out          assert dn_2._get_user_properties() == {"foo": "bar"}        def test_new_parquet_data_node_with_existing_file_is_ready_for_reading(self, parquet_file_path):          not_ready_dn_cfg = Config.configure_data_node(              "not_ready_data_node_config_id", "parquet", path="NOT_EXISTING.parquet"          )          not_ready_dn = _DataManager._bulk_get_or_create([not_ready_dn_cfg])[not_ready_dn_cfg]          assert not not_ready_dn.is_ready_for_reading            ready_dn_cfg = Config.configure_data_node("ready_data_node_config_id", "parquet", path=parquet_file_path)          ready_dn = _DataManager._bulk_get_or_create([ready_dn_cfg])[ready_dn_cfg]          assert ready_dn.is_ready_for_reading        @pytest.mark.parametrize(          ["properties", "exists"],          [              ({}, False),              ({"default_data": {"a": ["foo", "bar"]}}, True),          ],      )      def test_create_with_default_data(self, properties, exists):          dn = ParquetDataNode("foo", Scope.SCENARIO, DataNodeId("dn_id"), properties=properties)          assert os.path.exists(dn.path) is exists        @pytest.mark.parametrize("engine", __engine)      def test_read_file(self, engine, parquet_file_path):          not_existing_parquet = ParquetDataNode(              "foo", Scope.SCENARIO, properties={"path": "nonexistent.parquet", "engine": engine}          )          with pytest.raises(NoData):              assert not_existing_parquet.read() is None              not_existing_parquet.read_or_raise()            df = pd.read_parquet(parquet_file_path)          # Create ParquetDataNode without exposed_type (Default is pandas.DataFrame)          parquet_data_node_as_pandas = ParquetDataNode(              "bar", Scope.SCENARIO, properties={"path": parquet_file_path, "engine": engine}          )          data_pandas = parquet_data_node_as_pandas.read()          assert isinstance(data_pandas, pd.DataFrame)          assert len(data_pandas) == 2          assert data_pandas.equals(df)          assert np.array_equal(data_pandas.to_numpy(), df.to_numpy())            # Create ParquetDataNode with modin exposed_type          parquet_data_node_as_modin = ParquetDataNode(              "bar", Scope.SCENARIO, properties={"path": parquet_file_path, "exposed_type": "modin", "engine": engine}          )          data_modin = parquet_data_node_as_modin.read()          assert isinstance(data_modin, modin_pd.DataFrame)          assert len(data_modin) == 2          assert data_modin.equals(df)          assert np.array_equal(data_modin.to_numpy(), df.to_numpy())            # Create ParquetDataNode with numpy exposed_type          parquet_data_node_as_numpy = ParquetDataNode(              "bar", Scope.SCENARIO, properties={"path": parquet_file_path, "exposed_type": "numpy", "engine": engine}          )          data_numpy = parquet_data_node_as_numpy.read()          assert isinstance(data_numpy, np.ndarray)          assert len(data_numpy) == 2          assert np.array_equal(data_numpy, df.to_numpy())        @pytest.mark.parametrize("engine", __engine)      def test_read_folder(self, engine):          parquet_folder_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/parquet_example")            df = pd.read_parquet(parquet_folder_path)          parquet_data_node_as_pandas = ParquetDataNode(              "bar", Scope.SCENARIO, properties={"path": parquet_folder_path, "engine": engine}          )          data_pandas = parquet_data_node_as_pandas.read()          assert isinstance(data_pandas, pd.DataFrame)          assert len(data_pandas) == 5          assert data_pandas.equals(df)          assert np.array_equal(data_pandas.to_numpy(), df.to_numpy())        def test_set_path(self):          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": "foo.parquet"})          assert dn.path == "foo.parquet"          dn.path = "bar.parquet"          assert dn.path == "bar.parquet"        @pytest.mark.parametrize("engine", __engine)      def test_read_write_after_modify_path(self, engine):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.parquet")          new_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/temp.parquet")          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": path, "engine": engine})          read_data = dn.read()          assert read_data is not None          dn.path = new_path          with pytest.raises(FileNotFoundError):              dn.read()          dn.write(read_data)          assert dn.read().equals(read_data)        def test_read_custom_exposed_type(self):          example_parquet_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.parquet")            dn = ParquetDataNode(              "foo", Scope.SCENARIO, properties={"path": example_parquet_path, "exposed_type": MyCustomObject}          )          assert all([isinstance(obj, MyCustomObject) for obj in dn.read()])            dn = ParquetDataNode(              "foo", Scope.SCENARIO, properties={"path": example_parquet_path, "exposed_type": create_custom_class}          )          assert all([isinstance(obj, MyOtherCustomObject) for obj in dn.read()])        def test_raise_error_unknown_parquet_engine(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.parquet")          with pytest.raises(UnknownParquetEngine):              ParquetDataNode("foo", Scope.SCENARIO, properties={"path": path, "engine": "foo"})        def test_raise_error_unknown_compression_algorithm(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.parquet")          with pytest.raises(UnknownCompressionAlgorithm):              ParquetDataNode("foo", Scope.SCENARIO, properties={"path": path, "compression": "foo"})        def test_raise_error_invalid_exposed_type(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.parquet")          with pytest.raises(InvalidExposedType):              ParquetDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "foo"})        def test_read_empty_data(self, tmpdir_factory):          temp_file_path = str(tmpdir_factory.mktemp("data").join("temp.parquet"))          empty_df = pd.DataFrame([])          empty_df.to_parquet(temp_file_path)            # Pandas          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": temp_file_path, "exposed_type": "pandas"})          assert dn.read().equals(empty_df)            # Numpy          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": temp_file_path, "exposed_type": "numpy"})          assert np.array_equal(dn.read(), empty_df.to_numpy())            # Custom          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": temp_file_path, "exposed_type": MyCustomObject})          assert dn.read() == []        def test_get_system_file_modified_date_instead_of_last_edit_date(self, tmpdir_factory):          temp_file_path = str(tmpdir_factory.mktemp("data").join("temp.parquet"))          pd.DataFrame([]).to_parquet(temp_file_path)          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": temp_file_path, "exposed_type": "pandas"})            dn.write(pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}))          previous_edit_date = dn.last_edit_date            sleep(0.1)            pd.DataFrame(pd.DataFrame(data={"col1": [5, 6], "col2": [7, 8]})).to_parquet(temp_file_path)          new_edit_date = datetime.fromtimestamp(os.path.getmtime(temp_file_path))            assert previous_edit_date < dn.last_edit_date          assert new_edit_date == dn.last_edit_date            sleep(0.1)            dn.write(pd.DataFrame(data={"col1": [9, 10], "col2": [10, 12]}))          assert new_edit_date < dn.last_edit_date          os.unlink(temp_file_path)        def test_get_system_folder_modified_date_instead_of_last_edit_date(self, tmpdir_factory):          temp_folder_path = tmpdir_factory.mktemp("data").strpath          temp_file_path = os.path.join(temp_folder_path, "temp.parquet")          pd.DataFrame([]).to_parquet(temp_file_path)            dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": temp_folder_path})          initial_edit_date = dn.last_edit_date            # Sleep so that the file can be created successfully on Ubuntu          sleep(0.1)            pd.DataFrame(pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})).to_parquet(temp_file_path)          first_edit_date = datetime.fromtimestamp(os.path.getmtime(temp_file_path))          assert dn.last_edit_date > initial_edit_date          assert dn.last_edit_date == first_edit_date            sleep(0.1)            pd.DataFrame(pd.DataFrame(data={"col1": [5, 6], "col2": [7, 8]})).to_parquet(temp_file_path)          second_edit_date = datetime.fromtimestamp(os.path.getmtime(temp_file_path))          assert dn.last_edit_date > first_edit_date          assert dn.last_edit_date == second_edit_date            os.unlink(temp_file_path)        @pytest.mark.skipif(not util.find_spec("fastparquet"), reason="Append parquet requires fastparquet to be installed")      @pytest.mark.parametrize(          "content",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),              (pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}])),          ],      )      def test_append_pandas(self, parquet_file_path, default_data_frame, content):          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": parquet_file_path})          assert_frame_equal(dn.read(), default_data_frame)            dn.append(content)          assert_frame_equal(              dn.read(),              pd.concat([default_data_frame, pd.DataFrame(content, columns=["a", "b", "c"])]).reset_index(drop=True),          )        @pytest.mark.skipif(not util.find_spec("fastparquet"), reason="Append parquet requires fastparquet to be installed")      @pytest.mark.parametrize(          "content",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),              (pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}])),          ],      )      def test_append_modin(self, parquet_file_path, default_data_frame, content):          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": parquet_file_path, "exposed_type": "modin"})          df_equals(dn.read(), modin_pd.DataFrame(default_data_frame))            dn.append(content)          df_equals(              dn.read(),              modin_pd.concat([default_data_frame, pd.DataFrame(content, columns=["a", "b", "c"])]).reset_index(                  drop=True              ),          )        @pytest.mark.parametrize(          "data",          [              [{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}],              pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),              modin_pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),          ],      )      def test_write_to_disk(self, tmpdir_factory, data):          temp_file_path = str(tmpdir_factory.mktemp("data").join("temp.parquet"))          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": temp_file_path})          dn.write(data)            assert pathlib.Path(temp_file_path).exists()          assert isinstance(dn.read(), pd.DataFrame)        def test_filter_pandas_exposed_type(self, parquet_file_path):          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": parquet_file_path, "exposed_type": "pandas"})          dn.write(              [                  {"foo": 1, "bar": 1},                  {"foo": 1, "bar": 2},                  {"foo": 1},                  {"foo": 2, "bar": 2},                  {"bar": 2},              ]          )            # Test datanode indexing and slicing          assert dn["foo"].equals(pd.Series([1, 1, 1, 2, None]))          assert dn["bar"].equals(pd.Series([1, 2, None, 2, 2]))          assert dn[:2].equals(pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))            # Test filter data          filtered_by_filter_method = dn.filter(("foo", 1, Operator.EQUAL))          filtered_by_indexing = dn[dn["foo"] == 1]          expected_data = pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}, {"foo": 1.0}])          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("foo", 1, Operator.NOT_EQUAL))          filtered_by_indexing = dn[dn["foo"] != 1]          expected_data = pd.DataFrame([{"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("bar", 2, Operator.EQUAL))          filtered_by_indexing = dn[dn["bar"] == 2]          expected_data = pd.DataFrame([{"foo": 1.0, "bar": 2.0}, {"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)          filtered_by_indexing = dn[(dn["bar"] == 1) | (dn["bar"] == 2)]          expected_data = pd.DataFrame(              [                  {"foo": 1.0, "bar": 1.0},                  {"foo": 1.0, "bar": 2.0},                  {"foo": 2.0, "bar": 2.0},                  {"bar": 2.0},              ]          )          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)        def test_filter_modin_exposed_type(self, parquet_file_path):          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": parquet_file_path, "exposed_type": "modin"})          dn.write(              [                  {"foo": 1, "bar": 1},                  {"foo": 1, "bar": 2},                  {"foo": 1},                  {"foo": 2, "bar": 2},                  {"bar": 2},              ]          )            # Test datanode indexing and slicing          assert dn["foo"].equals(modin_pd.Series([1, 1, 1, 2, None]))          assert dn["bar"].equals(modin_pd.Series([1, 2, None, 2, 2]))          assert dn[:2].equals(modin_pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))            # Test filter data          filtered_by_filter_method = dn.filter(("foo", 1, Operator.EQUAL))          filtered_by_indexing = dn[dn["foo"] == 1]          expected_data = modin_pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}, {"foo": 1.0}])          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("foo", 1, Operator.NOT_EQUAL))          filtered_by_indexing = dn[dn["foo"] != 1]          expected_data = modin_pd.DataFrame([{"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("bar", 2, Operator.EQUAL))          filtered_by_indexing = dn[dn["bar"] == 2]          expected_data = modin_pd.DataFrame([{"foo": 1.0, "bar": 2.0}, {"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)          filtered_by_indexing = dn[(dn["bar"] == 1) | (dn["bar"] == 2)]          expected_data = modin_pd.DataFrame(              [                  {"foo": 1.0, "bar": 1.0},                  {"foo": 1.0, "bar": 2.0},                  {"foo": 2.0, "bar": 2.0},                  {"bar": 2.0},              ]          )          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)        def test_filter_numpy_exposed_type(self, parquet_file_path):          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": parquet_file_path, "exposed_type": "numpy"})          dn.write(              [                  [1, 1],                  [1, 2],                  [1, 3],                  [2, 1],                  [2, 2],                  [2, 3],              ]          )            # Test datanode indexing and slicing          assert np.array_equal(dn[0], np.array([1, 1]))          assert np.array_equal(dn[1], np.array([1, 2]))          assert np.array_equal(dn[:3], np.array([[1, 1], [1, 2], [1, 3]]))          assert np.array_equal(dn[:, 0], np.array([1, 1, 1, 2, 2, 2]))          assert np.array_equal(dn[1:4, :1], np.array([[1], [1], [2]]))            # Test filter data          assert np.array_equal(dn.filter((0, 1, Operator.EQUAL)), np.array([[1, 1], [1, 2], [1, 3]]))          assert np.array_equal(dn[dn[:, 0] == 1], np.array([[1, 1], [1, 2], [1, 3]]))            assert np.array_equal(dn.filter((0, 1, Operator.NOT_EQUAL)), np.array([[2, 1], [2, 2], [2, 3]]))          assert np.array_equal(dn[dn[:, 0] != 1], np.array([[2, 1], [2, 2], [2, 3]]))            assert np.array_equal(dn.filter((1, 2, Operator.EQUAL)), np.array([[1, 2], [2, 2]]))          assert np.array_equal(dn[dn[:, 1] == 2], np.array([[1, 2], [2, 2]]))            assert np.array_equal(              dn.filter([(1, 1, Operator.EQUAL), (1, 2, Operator.EQUAL)], JoinOperator.OR),              np.array([[1, 1], [1, 2], [2, 1], [2, 2]]),          )          assert np.array_equal(dn[(dn[:, 1] == 1) | (dn[:, 1] == 2)], np.array([[1, 1], [1, 2], [2, 1], [2, 2]]))        @pytest.mark.parametrize("engine", __engine)      def test_pandas_parquet_config_kwargs(self, engine, tmpdir_factory):          read_kwargs = {"filters": [("integer", "<", 10)], "columns": ["integer"]}          temp_file_path = str(tmpdir_factory.mktemp("data").join("temp.parquet"))          dn = ParquetDataNode(              "foo", Scope.SCENARIO, properties={"path": temp_file_path, "engine": engine, "read_kwargs": read_kwargs}          )            df = pd.read_csv(os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.csv"))          dn.write(df)            assert set(pd.read_parquet(temp_file_path).columns) == {"id", "integer", "text"}          assert set(dn.read().columns) == set(read_kwargs["columns"])            # !!! filter doesn't work with `fastparquet` without partition_cols          if engine == "pyarrow":              assert len(dn.read()) != len(df)              assert len(dn.read()) == 2        @pytest.mark.parametrize("engine", __engine)      def test_kwarg_precedence(self, engine, tmpdir_factory, default_data_frame):          # Precedence:          # 1. Class read/write methods          # 2. Defined in read_kwargs and write_kwargs, in properties          # 3. Defined top-level in properties            temp_file_path = str(tmpdir_factory.mktemp("data").join("temp.parquet"))          temp_file_2_path = str(tmpdir_factory.mktemp("data").join("temp_2.parquet"))          df = default_data_frame.copy(deep=True)            # Write          # 3          comp3 = "snappy"          dn = ParquetDataNode(              "foo", Scope.SCENARIO, properties={"path": temp_file_path, "engine": engine, "compression": comp3}          )          dn.write(df)          df.to_parquet(path=temp_file_2_path, compression=comp3, engine=engine)          with open(temp_file_2_path, "rb") as tf:              with pathlib.Path(temp_file_path).open("rb") as f:                  assert f.read() == tf.read()            # 3 and 2          comp2 = "gzip"          dn = ParquetDataNode(              "foo",              Scope.SCENARIO,              properties={                  "path": temp_file_path,                  "engine": engine,                  "compression": comp3,                  "write_kwargs": {"compression": comp2},              },          )          dn.write(df)          df.to_parquet(path=temp_file_2_path, compression=comp2, engine=engine)          with open(temp_file_2_path, "rb") as tf:              with pathlib.Path(temp_file_path).open("rb") as f:                  assert f.read() == tf.read()            # 3, 2 and 1          comp1 = "brotli"          dn = ParquetDataNode(              "foo",              Scope.SCENARIO,              properties={                  "path": temp_file_path,                  "engine": engine,                  "compression": comp3,                  "write_kwargs": {"compression": comp2},              },          )          dn.write_with_kwargs(df, compression=comp1)          df.to_parquet(path=temp_file_2_path, compression=comp1, engine=engine)          with open(temp_file_2_path, "rb") as tf:              with pathlib.Path(temp_file_path).open("rb") as f:                  assert f.read() == tf.read()            # Read          df.to_parquet(temp_file_path, engine=engine)          # 2          cols2 = ["a", "b"]          dn = ParquetDataNode(              "foo",              Scope.SCENARIO,              properties={"path": temp_file_path, "engine": engine, "read_kwargs": {"columns": cols2}},          )          assert set(dn.read().columns) == set(cols2)            # 1          cols1 = ["a"]          dn = ParquetDataNode(              "foo",              Scope.SCENARIO,              properties={"path": temp_file_path, "engine": engine, "read_kwargs": {"columns": cols2}},          )          assert set(dn.read_with_kwargs(columns=cols1).columns) == set(cols1)        def test_partition_cols(self, tmpdir_factory, default_data_frame: pd.DataFrame):          temp_dir_path = str(tmpdir_factory.mktemp("data").join("temp_dir"))            write_kwargs = {"partition_cols": ["a", "b"]}          dn = ParquetDataNode(              "foo", Scope.SCENARIO, properties={"path": temp_dir_path, "write_kwargs": write_kwargs}          )  # type: ignore          dn.write(default_data_frame)            assert pathlib.Path(temp_dir_path).is_dir()          # dtypes change during round-trip with partition_cols          pd.testing.assert_frame_equal(              dn.read().sort_index(axis=1),              default_data_frame.sort_index(axis=1),              check_dtype=False,              check_categorical=False,          )        def test_read_with_kwargs_never_written(self):          path = "data/node/path"          dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": path})          assert dn.read_with_kwargs() is None