import os  import pathlib  from datetime import datetime  from time import sleep  from typing import Dict    import modin.pandas as modin_pd  import numpy as np  import pandas as pd  import pytest  from modin.pandas.test.utils import df_equals  from pandas.testing import assert_frame_equal    from src.taipy.core.data._data_manager import _DataManager  from src.taipy.core.data.data_node_id import DataNodeId  from src.taipy.core.data.excel import ExcelDataNode  from src.taipy.core.data.operator import JoinOperator, Operator  from src.taipy.core.exceptions.exceptions import (      ExposedTypeLengthMismatch,      InvalidExposedType,      NoData,      NonExistingExcelSheet,      SheetNameLengthMismatch,  )  from taipy.config.common.scope import Scope  from taipy.config.config import Config      @pytest.fixture(scope="function", autouse=True)  def cleanup():      yield      path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/temp.xlsx")      if os.path.exists(path):          os.remove(path)      class MyCustomObject:      def __init__(self, id, integer, text):          self.id = id          self.integer = integer          self.text = text      class MyCustomObject1:      def __init__(self, id, integer, text):          self.id = id          self.integer = integer          self.text = text      class MyCustomObject2:      def __init__(self, id, integer, text):          self.id = id          self.integer = integer          self.text = text      class TestExcelDataNode:      def test_new_excel_data_node_with_existing_file_is_ready_for_reading(self):          not_ready_dn_cfg = Config.configure_data_node("not_ready_data_node_config_id", "excel", path="NOT_EXISTING.csv")          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")          ready_dn_cfg = Config.configure_data_node("ready_data_node_config_id", "excel", path=path)            dns = _DataManager._bulk_get_or_create([not_ready_dn_cfg, ready_dn_cfg])            assert not dns[not_ready_dn_cfg].is_ready_for_reading          assert dns[ready_dn_cfg].is_ready_for_reading        def test_create(self):          path = "data/node/path"          sheet_names = ["sheet_name_1", "sheet_name_2"]          dn = ExcelDataNode(              "foo_bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "sheet_name": sheet_names, "name": "super name"},          )          assert isinstance(dn, ExcelDataNode)          assert dn.storage_type() == "excel"          assert dn.config_id == "foo_bar"          assert dn.name == "super name"          assert dn.scope == Scope.SCENARIO          assert dn.id is not None          assert dn.owner_id is None          assert dn.parent_ids == set()          assert dn.last_edit_date is None          assert dn.job_ids == []          assert not dn.is_ready_for_reading          assert dn.path == path          assert dn.has_header is False          assert dn.sheet_name == sheet_names        def test_get_user_properties(self, excel_file):          dn_1 = ExcelDataNode("dn_1", Scope.SCENARIO, properties={"path": "data/node/path"})          assert dn_1._get_user_properties() == {}            dn_2 = ExcelDataNode(              "dn_2",              Scope.SCENARIO,              properties={                  "exposed_type": "numpy",                  "default_data": "foo",                  "default_path": excel_file,                  "has_header": False,                  "sheet_name": ["sheet_name_1", "sheet_name_2"],                  "foo": "bar",              },          )            # exposed_type, default_data, default_path, path, has_header are filtered out          assert dn_2._get_user_properties() == {"foo": "bar"}        def test_read_with_header(self):          with pytest.raises(NoData):              not_existing_excel = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": "WRONG.xlsx"})              assert not_existing_excel.read() is None              not_existing_excel.read_or_raise()            empty_excel_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/empty.xlsx")          empty_excel = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": empty_excel_path, "exposed_type": MyCustomObject, "has_header": True},          )          assert len(empty_excel.read()) == 0            path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")            # Create ExcelDataNode without exposed_type (Default is pandas.DataFrame)          excel_data_node_as_pandas = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "sheet_name": "Sheet1"}          )            data_pandas = excel_data_node_as_pandas.read()          assert isinstance(data_pandas, pd.DataFrame)          assert len(data_pandas) == 5          assert np.array_equal(data_pandas.to_numpy(), pd.read_excel(path).to_numpy())            # Create ExcelDataNode with modin exposed_type          excel_data_node_as_modin = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "sheet_name": "Sheet1", "exposed_type": "modin"}          )            data_modin = excel_data_node_as_modin.read()          assert isinstance(data_modin, modin_pd.DataFrame)          assert len(data_modin) == 5          assert np.array_equal(data_modin.to_numpy(), pd.read_excel(path).to_numpy())            # Create ExcelDataNode with numpy exposed_type          excel_data_node_as_numpy = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "exposed_type": "numpy", "sheet_name": "Sheet1"}          )            data_numpy = excel_data_node_as_numpy.read()          assert isinstance(data_numpy, np.ndarray)          assert len(data_numpy) == 5          assert np.array_equal(data_numpy, pd.read_excel(path).to_numpy())            # Create the same ExcelDataNode but with custom exposed_type          non_existing_sheet_name_custom = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "sheet_name": "abc", "exposed_type": MyCustomObject},          )          with pytest.raises(NonExistingExcelSheet):              non_existing_sheet_name_custom.read()            excel_data_node_as_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "exposed_type": MyCustomObject, "sheet_name": "Sheet1"},          )            data_custom = excel_data_node_as_custom_object.read()          assert isinstance(data_custom, list)          assert len(data_custom) == 5            for (_, row_pandas), row_custom in zip(data_pandas.iterrows(), data_custom):              assert isinstance(row_custom, MyCustomObject)              assert row_pandas["id"] == row_custom.id              assert row_pandas["integer"] == row_custom.integer              assert row_pandas["text"] == row_custom.text        def test_read_without_header(self):          not_existing_excel = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": "WRONG.xlsx", "has_header": False}          )          with pytest.raises(NoData):              assert not_existing_excel.read() is None              not_existing_excel.read_or_raise()            path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")            # Create ExcelDataNode without exposed_type (Default is pandas.DataFrame)          excel_data_node_as_pandas = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "has_header": False, "sheet_name": "Sheet1"}          )          data_pandas = excel_data_node_as_pandas.read()          assert isinstance(data_pandas, pd.DataFrame)          assert len(data_pandas) == 6          assert np.array_equal(data_pandas.to_numpy(), pd.read_excel(path, header=None).to_numpy())            # Create ExcelDataNode with modin exposed_type          excel_data_node_as_modin = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "sheet_name": "Sheet1", "exposed_type": "modin"},          )          data_modin = excel_data_node_as_modin.read()          assert isinstance(data_modin, modin_pd.DataFrame)          assert len(data_modin) == 6          assert np.array_equal(data_modin.to_numpy(), pd.read_excel(path, header=None).to_numpy())            # Create ExcelDataNode with numpy exposed_type          excel_data_node_as_numpy = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "exposed_type": "numpy", "sheet_name": "Sheet1"},          )            data_numpy = excel_data_node_as_numpy.read()          assert isinstance(data_numpy, np.ndarray)          assert len(data_numpy) == 6          assert np.array_equal(data_numpy, pd.read_excel(path, header=None).to_numpy())            # Create the same ExcelDataNode but with custom exposed_type          non_existing_sheet_name_custom = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "sheet_name": "abc", "exposed_type": MyCustomObject},          )          with pytest.raises(NonExistingExcelSheet):              non_existing_sheet_name_custom.read()            excel_data_node_as_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={                  "path": path,                  "has_header": False,                  "exposed_type": MyCustomObject,                  "sheet_name": "Sheet1",              },          )            data_custom = excel_data_node_as_custom_object.read()          assert isinstance(data_custom, list)          assert len(data_custom) == 6            for (_, row_pandas), row_custom in zip(data_pandas.iterrows(), data_custom):              assert isinstance(row_custom, MyCustomObject)              assert row_pandas[0] == row_custom.id              assert row_pandas[1] == row_custom.integer              assert row_pandas[2] == row_custom.text        @pytest.mark.parametrize(          "content,columns",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}], None),              ([[11, 22, 33], [44, 55, 66]], None),              ([[11, 22, 33], [44, 55, 66]], ["e", "f", "g"]),          ],      )      def test_write(self, excel_file, default_data_frame, content, columns):          excel_dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file, "sheet_name": "Sheet1"})          assert np.array_equal(excel_dn.read().values, default_data_frame.values)          if not columns:              excel_dn.write(content)              df = pd.DataFrame(content)          else:              excel_dn.write_with_column_names(content, columns)              df = pd.DataFrame(content, columns=columns)            assert np.array_equal(excel_dn.read().values, df.values)            excel_dn.write(None)          assert len(excel_dn.read()) == 0        @pytest.mark.parametrize(          "content,sheet_name",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}], "sheet_name"),              ([[11, 22, 33], [44, 55, 66]], ["sheet_name"]),          ],      )      def test_write_with_sheet_name(self, excel_file_with_sheet_name, default_data_frame, content, sheet_name):          excel_dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file_with_sheet_name, "sheet_name": sheet_name}          )          df = pd.DataFrame(content)            if isinstance(sheet_name, str):              assert np.array_equal(excel_dn.read().values, default_data_frame.values)          else:              assert np.array_equal(excel_dn.read()["sheet_name"].values, default_data_frame.values)            excel_dn.write(content)          if isinstance(sheet_name, str):              assert np.array_equal(excel_dn.read().values, df.values)          else:              assert np.array_equal(excel_dn.read()["sheet_name"].values, df.values)            sheet_names = pd.ExcelFile(excel_file_with_sheet_name).sheet_names          expected_sheet_name = sheet_name[0] if isinstance(sheet_name, list) else sheet_name            assert sheet_names[0] == expected_sheet_name            excel_dn.write(None)          if isinstance(sheet_name, str):              assert len(excel_dn.read()) == 0          else:              assert len(excel_dn.read()) == 1        @pytest.mark.parametrize(          "content,sheet_name",          [              ([[11, 22, 33], [44, 55, 66]], ["sheet_name_1", "sheet_name_2"]),          ],      )      def test_raise_write_with_sheet_name_length_mismatch(          self, excel_file_with_sheet_name, default_data_frame, content, sheet_name      ):          excel_dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file_with_sheet_name, "sheet_name": sheet_name}          )          with pytest.raises(SheetNameLengthMismatch):              excel_dn.write(content)        @pytest.mark.parametrize(          "content",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),          ],      )      def test_write_without_sheet_name(self, excel_file_with_sheet_name, default_data_frame, content):          excel_dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file_with_sheet_name})          default_data_frame = {"sheet_name": default_data_frame}          df = {"Sheet1": pd.DataFrame(content)}            assert np.array_equal(excel_dn.read()["sheet_name"].values, default_data_frame["sheet_name"].values)            excel_dn.write(content)          assert np.array_equal(excel_dn.read()["Sheet1"].values, df["Sheet1"].values)            sheet_names = pd.ExcelFile(excel_file_with_sheet_name).sheet_names          expected_sheet_name = "Sheet1"            assert sheet_names[0] == expected_sheet_name            excel_dn.write(None)          assert len(excel_dn.read()) == 1        @pytest.mark.parametrize(          "content,columns,sheet_name",          [              ([[11, 22, 33], [44, 55, 66]], ["e", "f", "g"], "sheet_name"),              ([[11, 22, 33], [44, 55, 66]], ["e", "f", "g"], ["sheet_name"]),          ],      )      def test_write_with_column_and_sheet_name(          self, excel_file_with_sheet_name, default_data_frame, content, columns, sheet_name      ):          excel_dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file_with_sheet_name, "sheet_name": sheet_name}          )          df = pd.DataFrame(content)            if isinstance(sheet_name, str):              assert np.array_equal(excel_dn.read().values, default_data_frame.values)          else:              assert np.array_equal(excel_dn.read()["sheet_name"].values, default_data_frame.values)            excel_dn.write_with_column_names(content, columns)            if isinstance(sheet_name, str):              assert np.array_equal(excel_dn.read().values, df.values)          else:              assert np.array_equal(excel_dn.read()["sheet_name"].values, df.values)            sheet_names = pd.ExcelFile(excel_file_with_sheet_name).sheet_names          expected_sheet_name = sheet_name[0] if isinstance(sheet_name, list) else sheet_name            assert sheet_names[0] == expected_sheet_name            excel_dn.write(None)          if isinstance(sheet_name, str):              assert len(excel_dn.read()) == 0          else:              assert len(excel_dn.read()) == 1        @pytest.mark.parametrize(          "content,columns",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}], None),              ([[11, 22, 33], [44, 55, 66]], None),              ([[11, 22, 33], [44, 55, 66]], ["e", "f", "g"]),          ],      )      def test_write_modin(self, excel_file, default_data_frame, content, columns):          excel_dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file, "sheet_name": "Sheet1", "exposed_type": "modin"}          )          assert np.array_equal(excel_dn.read().values, default_data_frame.values)          if not columns:              excel_dn.write(content)              df = modin_pd.DataFrame(content)          else:              excel_dn.write_with_column_names(content, columns)              df = modin_pd.DataFrame(content, columns=columns)            assert np.array_equal(excel_dn.read().values, df.values)            excel_dn.write(None)          assert len(excel_dn.read()) == 0        def test_read_multi_sheet_with_header(self):          not_existing_excel = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": "WRONG.xlsx", "sheet_name": ["sheet_name_1", "sheet_name_2"]},          )          with pytest.raises(NoData):              assert not_existing_excel.read() is None              not_existing_excel.read_or_raise()            path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")          sheet_names = ["Sheet1", "Sheet2"]            # Create ExcelDataNode without exposed_type (Default is pandas.DataFrame)          excel_data_node_as_pandas = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "sheet_name": sheet_names}          )            data_pandas = excel_data_node_as_pandas.read()          assert isinstance(data_pandas, Dict)          assert len(data_pandas) == 2          assert all(              len(data_pandas[sheet_name] == 5) and isinstance(data_pandas[sheet_name], pd.DataFrame)              for sheet_name in sheet_names          )          assert list(data_pandas.keys()) == sheet_names          for sheet_name in sheet_names:              assert data_pandas[sheet_name].equals(pd.read_excel(path, sheet_name=sheet_name))            excel_data_node_as_pandas_no_sheet_name = ExcelDataNode("bar", Scope.SCENARIO, properties={"path": path})            data_pandas_no_sheet_name = excel_data_node_as_pandas_no_sheet_name.read()          assert isinstance(data_pandas_no_sheet_name, Dict)          for key in data_pandas_no_sheet_name.keys():              assert isinstance(data_pandas_no_sheet_name[key], pd.DataFrame)              assert data_pandas[key].equals(data_pandas_no_sheet_name[key])            # Create ExcelDataNode with modin exposed_type          excel_data_node_as_modin = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "sheet_name": sheet_names, "exposed_type": "modin"}          )            data_modin = excel_data_node_as_modin.read()          assert isinstance(data_modin, Dict)          assert len(data_modin) == 2          assert all(              len(data_modin[sheet_name] == 5) and isinstance(data_modin[sheet_name], modin_pd.DataFrame)              for sheet_name in sheet_names          )          assert list(data_modin.keys()) == sheet_names          for sheet_name in sheet_names:              assert data_modin[sheet_name].equals(modin_pd.read_excel(path, sheet_name=sheet_name))            excel_data_node_as_pandas_no_sheet_name = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "exposed_type": "modin"}          )            data_modin_no_sheet_name = excel_data_node_as_pandas_no_sheet_name.read()          assert isinstance(data_modin_no_sheet_name, Dict)          for key in data_modin_no_sheet_name.keys():              assert isinstance(data_modin_no_sheet_name[key], modin_pd.DataFrame)              assert data_modin[key].equals(data_modin_no_sheet_name[key])            # Create ExcelDataNode with numpy exposed_type          excel_data_node_as_numpy = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "sheet_name": sheet_names, "exposed_type": "numpy"},          )            data_numpy = excel_data_node_as_numpy.read()          assert isinstance(data_numpy, Dict)          assert len(data_numpy) == 2          assert all(              len(data_numpy[sheet_name] == 5) and isinstance(data_numpy[sheet_name], np.ndarray)              for sheet_name in sheet_names          )          assert list(data_numpy.keys()) == sheet_names          for sheet_name in sheet_names:              assert np.array_equal(data_pandas[sheet_name], pd.read_excel(path, sheet_name=sheet_name).to_numpy())            excel_data_node_as_numpy_no_sheet_name = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "exposed_type": "numpy"},          )            data_numpy_no_sheet_name = excel_data_node_as_numpy_no_sheet_name.read()          assert isinstance(data_numpy_no_sheet_name, Dict)          for key in data_numpy_no_sheet_name.keys():              assert isinstance(data_numpy_no_sheet_name[key], np.ndarray)              assert np.array_equal(data_numpy[key], data_numpy_no_sheet_name[key])            # Create the same ExcelDataNode but with custom exposed_type          non_existing_sheet_name_custom = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={                  "path": path,                  "sheet_name": ["Sheet1", "xyz"],                  "exposed_type": MyCustomObject1,              },          )          with pytest.raises(NonExistingExcelSheet):              non_existing_sheet_name_custom.read()            excel_data_node_as_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "sheet_name": sheet_names, "exposed_type": MyCustomObject1},          )            data_custom = excel_data_node_as_custom_object.read()          assert isinstance(data_custom, Dict)          assert len(data_custom) == 2          assert all(len(data_custom[sheet_name]) == 5 for sheet_name in sheet_names)          assert list(data_custom.keys()) == sheet_names            for sheet_name in sheet_names:              sheet_data_pandas, sheet_data_custom = data_pandas[sheet_name], data_custom[sheet_name]              for (_, row_pandas), row_custom in zip(sheet_data_pandas.iterrows(), sheet_data_custom):                  assert isinstance(row_custom, MyCustomObject1)                  assert row_pandas["id"] == row_custom.id                  assert row_pandas["integer"] == row_custom.integer                  assert row_pandas["text"] == row_custom.text            excel_data_node_as_custom_object_no_sheet_name = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "exposed_type": MyCustomObject1},          )            data_custom_no_sheet_name = excel_data_node_as_custom_object_no_sheet_name.read()          assert isinstance(data_custom_no_sheet_name, Dict)          assert len(data_custom_no_sheet_name) == 2          assert data_custom.keys() == data_custom_no_sheet_name.keys()            for sheet_name in sheet_names:              sheet_data_custom_no_sheet_name, sheet_data_custom = (                  data_custom_no_sheet_name[sheet_name],                  data_custom[sheet_name],              )              for row_custom_no_sheet_name, row_custom in zip(sheet_data_custom_no_sheet_name, sheet_data_custom):                  assert isinstance(row_custom_no_sheet_name, MyCustomObject1)                  assert row_custom_no_sheet_name.id == row_custom.id                  assert row_custom_no_sheet_name.integer == row_custom.integer                  assert row_custom_no_sheet_name.text == row_custom.text            with pytest.raises(ExposedTypeLengthMismatch):              dn = ExcelDataNode(                  "bar",                  Scope.SCENARIO,                  properties={                      "path": path,                      "sheet_name": ["Sheet1"],                      "exposed_type": [MyCustomObject1, MyCustomObject2],                  },              )              dn.read()            custom_class_dict = {"Sheet1": MyCustomObject1, "Sheet2": MyCustomObject2}            excel_data_node_as_multi_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "sheet_name": sheet_names, "exposed_type": custom_class_dict},          )          assert excel_data_node_as_multi_custom_object.properties["exposed_type"] == custom_class_dict            excel_data_node_as_multi_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "sheet_name": sheet_names, "exposed_type": [MyCustomObject1, MyCustomObject2]},          )          assert excel_data_node_as_multi_custom_object.properties["exposed_type"] == [MyCustomObject1, MyCustomObject2]            multi_data_custom = excel_data_node_as_multi_custom_object.read()          assert isinstance(multi_data_custom, Dict)          assert len(multi_data_custom) == 2          assert all(len(multi_data_custom[sheet_name]) == 5 for sheet_name in sheet_names)          assert list(multi_data_custom.keys()) == sheet_names            for sheet_name, custom_class in custom_class_dict.items():              sheet_data_pandas, sheet_data_custom = data_pandas[sheet_name], multi_data_custom[sheet_name]              for (_, row_pandas), row_custom in zip(sheet_data_pandas.iterrows(), sheet_data_custom):                  assert isinstance(row_custom, custom_class)                  assert row_pandas["id"] == row_custom.id                  assert row_pandas["integer"] == row_custom.integer                  assert row_pandas["text"] == row_custom.text            excel_data_node_as_multi_custom_object_no_sheet_name = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "exposed_type": custom_class_dict},          )          assert excel_data_node_as_multi_custom_object_no_sheet_name.properties["exposed_type"] == custom_class_dict            multi_data_custom_no_sheet_name = excel_data_node_as_multi_custom_object_no_sheet_name.read()          assert isinstance(multi_data_custom_no_sheet_name, Dict)          assert len(multi_data_custom_no_sheet_name) == 2          assert multi_data_custom.keys() == multi_data_custom_no_sheet_name.keys()            for sheet_name, custom_class in custom_class_dict.items():              sheet_data_custom_no_sheet_name, sheet_data_custom = (                  multi_data_custom_no_sheet_name[sheet_name],                  multi_data_custom[sheet_name],              )              for row_custom_no_sheet_name, row_custom in zip(sheet_data_custom_no_sheet_name, sheet_data_custom):                  assert isinstance(row_custom_no_sheet_name, custom_class)                  assert row_custom_no_sheet_name.id == row_custom.id                  assert row_custom_no_sheet_name.integer == row_custom.integer                  assert row_custom_no_sheet_name.text == row_custom.text        def test_read_multi_sheet_without_header(self):          not_existing_excel = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": "WRONG.xlsx", "has_header": False, "sheet_name": ["sheet_name_1", "sheet_name_2"]},          )          with pytest.raises(NoData):              assert not_existing_excel.read() is None              not_existing_excel.read_or_raise()            path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")          sheet_names = ["Sheet1", "Sheet2"]            # Create ExcelDataNode without exposed_type (Default is pandas.DataFrame)          excel_data_node_as_pandas = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "has_header": False, "sheet_name": sheet_names}          )          data_pandas = excel_data_node_as_pandas.read()          assert isinstance(data_pandas, Dict)          assert len(data_pandas) == 2          assert all(len(data_pandas[sheet_name]) == 6 for sheet_name in sheet_names)          assert list(data_pandas.keys()) == sheet_names          for sheet_name in sheet_names:              assert isinstance(data_pandas[sheet_name], pd.DataFrame)              assert data_pandas[sheet_name].equals(pd.read_excel(path, header=None, sheet_name=sheet_name))            excel_data_node_as_pandas_no_sheet_name = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "has_header": False}          )          data_pandas_no_sheet_name = excel_data_node_as_pandas_no_sheet_name.read()          assert isinstance(data_pandas_no_sheet_name, Dict)          for key in data_pandas_no_sheet_name.keys():              assert isinstance(data_pandas_no_sheet_name[key], pd.DataFrame)              assert data_pandas[key].equals(data_pandas_no_sheet_name[key])            # Create ExcelDataNode with modin exposed_type          excel_data_node_as_modin = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "sheet_name": sheet_names, "exposed_type": "modin"},          )          data_modin = excel_data_node_as_modin.read()          assert isinstance(data_modin, Dict)          assert len(data_modin) == 2          assert all(len(data_modin[sheet_name]) == 6 for sheet_name in sheet_names)          assert list(data_modin.keys()) == sheet_names          for sheet_name in sheet_names:              assert isinstance(data_modin[sheet_name], modin_pd.DataFrame)              assert data_modin[sheet_name].equals(pd.read_excel(path, header=None, sheet_name=sheet_name))            excel_data_node_as_modin_no_sheet_name = ExcelDataNode(              "bar", Scope.SCENARIO, properties={"path": path, "has_header": False, "exposed_type": "modin"}          )          data_modin_no_sheet_name = excel_data_node_as_modin_no_sheet_name.read()          assert isinstance(data_modin_no_sheet_name, Dict)          for key in data_modin_no_sheet_name.keys():              assert isinstance(data_modin_no_sheet_name[key], modin_pd.DataFrame)              assert data_modin[key].equals(data_modin_no_sheet_name[key])            # Create ExcelDataNode with numpy exposed_type          excel_data_node_as_numpy = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "sheet_name": sheet_names, "exposed_type": "numpy"},          )            data_numpy = excel_data_node_as_numpy.read()          assert isinstance(data_numpy, Dict)          assert len(data_numpy) == 2          assert all(              len(data_numpy[sheet_name] == 6) and isinstance(data_numpy[sheet_name], np.ndarray)              for sheet_name in sheet_names          )          assert list(data_numpy.keys()) == sheet_names          for sheet_name in sheet_names:              assert np.array_equal(                  data_pandas[sheet_name], pd.read_excel(path, header=None, sheet_name=sheet_name).to_numpy()              )            excel_data_node_as_numpy_no_sheet_name = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "exposed_type": "numpy"},          )            data_numpy_no_sheet_name = excel_data_node_as_numpy_no_sheet_name.read()          assert isinstance(data_numpy_no_sheet_name, Dict)          for key in data_numpy_no_sheet_name.keys():              assert isinstance(data_numpy_no_sheet_name[key], np.ndarray)              assert np.array_equal(data_numpy[key], data_numpy_no_sheet_name[key])            # Create the same ExcelDataNode but with custom exposed_type          non_existing_sheet_name_custom = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={                  "path": path,                  "has_header": False,                  "sheet_name": ["Sheet1", "xyz"],                  "exposed_type": MyCustomObject1,              },          )          with pytest.raises(NonExistingExcelSheet):              non_existing_sheet_name_custom.read()            excel_data_node_as_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={                  "path": path,                  "has_header": False,                  "sheet_name": sheet_names,                  "exposed_type": MyCustomObject1,              },          )            data_custom = excel_data_node_as_custom_object.read()          assert excel_data_node_as_custom_object.exposed_type == MyCustomObject1          assert isinstance(data_custom, Dict)          assert len(data_custom) == 2          assert all(len(data_custom[sheet_name]) == 6 for sheet_name in sheet_names)          assert list(data_custom.keys()) == sheet_names            for sheet_name in sheet_names:              sheet_data_pandas, sheet_data_custom = data_pandas[sheet_name], data_custom[sheet_name]              for (_, row_pandas), row_custom in zip(sheet_data_pandas.iterrows(), sheet_data_custom):                  assert isinstance(row_custom, MyCustomObject1)                  assert row_pandas[0] == row_custom.id                  assert row_pandas[1] == row_custom.integer                  assert row_pandas[2] == row_custom.text            excel_data_node_as_custom_object_no_sheet_name = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "exposed_type": MyCustomObject1},          )            data_custom_no_sheet_name = excel_data_node_as_custom_object_no_sheet_name.read()          assert isinstance(data_custom_no_sheet_name, Dict)          assert len(data_custom_no_sheet_name) == 2          assert data_custom.keys() == data_custom_no_sheet_name.keys()            for sheet_name in sheet_names:              sheet_data_custom_no_sheet_name, sheet_data_custom = (                  data_custom_no_sheet_name[sheet_name],                  data_custom[sheet_name],              )              for row_custom_no_sheet_name, row_custom in zip(sheet_data_custom_no_sheet_name, sheet_data_custom):                  assert isinstance(row_custom_no_sheet_name, MyCustomObject1)                  assert row_custom_no_sheet_name.id == row_custom.id                  assert row_custom_no_sheet_name.integer == row_custom.integer                  assert row_custom_no_sheet_name.text == row_custom.text            with pytest.raises(ExposedTypeLengthMismatch):              dn = ExcelDataNode(                  "bar",                  Scope.SCENARIO,                  properties={                      "path": path,                      "sheet_name": ["Sheet1"],                      "exposed_type": [MyCustomObject1, MyCustomObject2],                      "has_header": False,                  },              )              dn.read()            custom_class_dict = {"Sheet1": MyCustomObject1, "Sheet2": MyCustomObject2}            excel_data_node_as_multi_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={                  "path": path,                  "sheet_name": sheet_names,                  "exposed_type": custom_class_dict,                  "has_header": False,              },          )          assert excel_data_node_as_multi_custom_object.properties["exposed_type"] == custom_class_dict            excel_data_node_as_multi_custom_object = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={                  "path": path,                  "sheet_name": sheet_names,                  "exposed_type": [MyCustomObject1, MyCustomObject2],                  "has_header": False,              },          )          assert excel_data_node_as_multi_custom_object.properties["exposed_type"] == [MyCustomObject1, MyCustomObject2]            multi_data_custom = excel_data_node_as_multi_custom_object.read()          assert isinstance(multi_data_custom, Dict)          assert len(multi_data_custom) == 2          assert all(len(multi_data_custom[sheet_name]) == 6 for sheet_name in sheet_names)          assert list(multi_data_custom.keys()) == sheet_names            for sheet_name, custom_class in custom_class_dict.items():              sheet_data_pandas, sheet_data_custom = data_pandas[sheet_name], multi_data_custom[sheet_name]              for (_, row_pandas), row_custom in zip(sheet_data_pandas.iterrows(), sheet_data_custom):                  assert isinstance(row_custom, custom_class)                  assert row_pandas[0] == row_custom.id                  assert row_pandas[1] == row_custom.integer                  assert row_pandas[2] == row_custom.text            excel_data_node_as_multi_custom_object_no_sheet_name = ExcelDataNode(              "bar",              Scope.SCENARIO,              properties={"path": path, "has_header": False, "exposed_type": custom_class_dict},          )            multi_data_custom_no_sheet_name = excel_data_node_as_multi_custom_object_no_sheet_name.read()          assert isinstance(multi_data_custom_no_sheet_name, Dict)          assert len(multi_data_custom_no_sheet_name) == 2          assert multi_data_custom.keys() == multi_data_custom_no_sheet_name.keys()            for sheet_name, custom_class in custom_class_dict.items():              sheet_data_custom_no_sheet_name, sheet_data_custom = (                  multi_data_custom_no_sheet_name[sheet_name],                  multi_data_custom[sheet_name],              )              for row_custom_no_sheet_name, row_custom in zip(sheet_data_custom_no_sheet_name, sheet_data_custom):                  assert isinstance(row_custom_no_sheet_name, custom_class)                  assert row_custom_no_sheet_name.id == row_custom.id                  assert row_custom_no_sheet_name.integer == row_custom.integer                  assert row_custom_no_sheet_name.text == row_custom.text        @pytest.mark.parametrize(          "content,columns",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}], None),              ([[11, 22, 33], [44, 55, 66]], None),              ([[11, 22, 33], [44, 55, 66]], ["e", "f", "g"]),          ],      )      def test_write_multi_sheet(self, excel_file_with_multi_sheet, default_multi_sheet_data_frame, content, columns):          sheet_names = ["Sheet1", "Sheet2"]            excel_dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": excel_file_with_multi_sheet, "sheet_name": sheet_names},          )            for sheet_name in sheet_names:              assert np.array_equal(excel_dn.read()[sheet_name].values, default_multi_sheet_data_frame[sheet_name].values)            multi_sheet_content = {sheet_name: pd.DataFrame(content) for sheet_name in sheet_names}            excel_dn.write(multi_sheet_content)            for sheet_name in sheet_names:              assert np.array_equal(excel_dn.read()[sheet_name].values, multi_sheet_content[sheet_name].values)        def test_write_multi_sheet_numpy(self, excel_file_with_multi_sheet):          sheet_names = ["Sheet1", "Sheet2"]            excel_dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": excel_file_with_multi_sheet, "sheet_name": sheet_names, "exposed_type": "numpy"},          )            sheets_data = [[11, 22, 33], [44, 55, 66]]          data = {              sheet_name: pd.DataFrame(sheet_data).to_numpy() for sheet_name, sheet_data in zip(sheet_names, sheets_data)          }          excel_dn.write(data)          read_data = excel_dn.read()          assert all(np.array_equal(data[sheet_name], read_data[sheet_name]) for sheet_name in sheet_names)        @pytest.mark.parametrize(          "content,columns",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}], None),              ([[11, 22, 33], [44, 55, 66]], None),              ([[11, 22, 33], [44, 55, 66]], ["e", "f", "g"]),          ],      )      def test_write_multi_sheet_with_modin(          self, excel_file_with_multi_sheet, default_multi_sheet_data_frame, content, columns      ):          sheet_names = ["Sheet1", "Sheet2"]            excel_dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": excel_file_with_multi_sheet, "sheet_name": sheet_names, "exposed_type": "modin"},          )            for sheet_name in sheet_names:              assert np.array_equal(excel_dn.read()[sheet_name].values, default_multi_sheet_data_frame[sheet_name].values)            multi_sheet_content = {sheet_name: modin_pd.DataFrame(content) for sheet_name in sheet_names}            excel_dn.write(multi_sheet_content)            for sheet_name in sheet_names:              assert np.array_equal(excel_dn.read()[sheet_name].values, multi_sheet_content[sheet_name].values)        @pytest.mark.parametrize(          "content",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),              (pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}])),              ([[11, 22, 33], [44, 55, 66]]),          ],      )      def test_append_pandas_with_sheetname(self, excel_file, default_data_frame, content):          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file, "sheet_name": "Sheet1"})          assert_frame_equal(dn.read(), default_data_frame)            dn.append(content)          assert_frame_equal(              dn.read(),              pd.concat([default_data_frame, pd.DataFrame(content, columns=["a", "b", "c"])]).reset_index(drop=True),          )        @pytest.mark.parametrize(          "content",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),              (pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}])),              ([[11, 22, 33], [44, 55, 66]]),          ],      )      def test_append_pandas_without_sheetname(self, excel_file, default_data_frame, content):          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file})          assert_frame_equal(dn.read()["Sheet1"], default_data_frame)            dn.append(content)          assert_frame_equal(              dn.read()["Sheet1"],              pd.concat([default_data_frame, pd.DataFrame(content, columns=["a", "b", "c"])]).reset_index(drop=True),          )        @pytest.mark.parametrize(          "content",          [              (                  {                      "Sheet1": pd.DataFrame([{"a": 11, "b": 22, "c": 33}]),                      "Sheet2": pd.DataFrame([{"a": 44, "b": 55, "c": 66}]),                  }              ),              (                  {                      "Sheet1": pd.DataFrame({"a": [11, 44], "b": [22, 55], "c": [33, 66]}),                      "Sheet2": pd.DataFrame([{"a": 77, "b": 88, "c": 99}]),                  }              ),              ({"Sheet1": np.array([[11, 22, 33], [44, 55, 66]]), "Sheet2": np.array([[77, 88, 99]])}),          ],      )      def test_append_pandas_multisheet(self, excel_file_with_multi_sheet, default_multi_sheet_data_frame, content):          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file_with_multi_sheet, "sheet_name": ["Sheet1", "Sheet2"]}          )          assert_frame_equal(dn.read()["Sheet1"], default_multi_sheet_data_frame["Sheet1"])          assert_frame_equal(dn.read()["Sheet2"], default_multi_sheet_data_frame["Sheet2"])            dn.append(content)            assert_frame_equal(              dn.read()["Sheet1"],              pd.concat(                  [default_multi_sheet_data_frame["Sheet1"], pd.DataFrame(content["Sheet1"], columns=["a", "b", "c"])]              ).reset_index(drop=True),          )          assert_frame_equal(              dn.read()["Sheet2"],              pd.concat(                  [default_multi_sheet_data_frame["Sheet2"], pd.DataFrame(content["Sheet2"], columns=["a", "b", "c"])]              ).reset_index(drop=True),          )        @pytest.mark.parametrize(          "content",          [              ({"Sheet1": pd.DataFrame([{"a": 11, "b": 22, "c": 33}])}),              (pd.DataFrame({"a": [11, 44], "b": [22, 55], "c": [33, 66]})),              ([[11, 22, 33], [44, 55, 66]]),          ],      )      def test_append_only_first_sheet_of_a_multisheet_file(          self, excel_file_with_multi_sheet, default_multi_sheet_data_frame, content      ):          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file_with_multi_sheet, "sheet_name": ["Sheet1", "Sheet2"]}          )          assert_frame_equal(dn.read()["Sheet1"], default_multi_sheet_data_frame["Sheet1"])          assert_frame_equal(dn.read()["Sheet2"], default_multi_sheet_data_frame["Sheet2"])            dn.append(content)            appended_content = content["Sheet1"] if isinstance(content, dict) else content          assert_frame_equal(              dn.read()["Sheet1"],              pd.concat(                  [default_multi_sheet_data_frame["Sheet1"], pd.DataFrame(appended_content, columns=["a", "b", "c"])]              ).reset_index(drop=True),          )          assert_frame_equal(dn.read()["Sheet2"], default_multi_sheet_data_frame["Sheet2"])        @pytest.mark.parametrize(          "content",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),              (modin_pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}])),              ([[11, 22, 33], [44, 55, 66]]),          ],      )      def test_append_modin_with_sheetname(self, excel_file, default_data_frame, content):          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file, "sheet_name": "Sheet1", "exposed_type": "modin"}          )          df_equals(dn.read(), modin_pd.DataFrame(default_data_frame))            dn.append(content)          df_equals(              dn.read(),              modin_pd.concat(                  [modin_pd.DataFrame(default_data_frame), modin_pd.DataFrame(content, columns=["a", "b", "c"])]              ).reset_index(drop=True),          )        @pytest.mark.parametrize(          "content",          [              ([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}]),              (modin_pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}])),              ([[11, 22, 33], [44, 55, 66]]),          ],      )      def test_append_modin_without_sheetname(self, excel_file, default_data_frame, content):          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file, "exposed_type": "modin"})          df_equals(dn.read()["Sheet1"], default_data_frame)            dn.append(content)          df_equals(              dn.read()["Sheet1"],              modin_pd.concat([default_data_frame, modin_pd.DataFrame(content, columns=["a", "b", "c"])]).reset_index(                  drop=True              ),          )        @pytest.mark.parametrize(          "content",          [              (                  {                      "Sheet1": modin_pd.DataFrame([{"a": 11, "b": 22, "c": 33}]),                      "Sheet2": modin_pd.DataFrame([{"a": 44, "b": 55, "c": 66}]),                  }              ),              (                  {                      "Sheet1": modin_pd.DataFrame({"a": [11, 44], "b": [22, 55], "c": [33, 66]}),                      "Sheet2": modin_pd.DataFrame([{"a": 77, "b": 88, "c": 99}]),                  }              ),              ({"Sheet1": np.array([[11, 22, 33], [44, 55, 66]]), "Sheet2": np.array([[77, 88, 99]])}),          ],      )      def test_append_modin_multisheet(self, excel_file_with_multi_sheet, default_multi_sheet_data_frame, content):          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={                  "path": excel_file_with_multi_sheet,                  "sheet_name": ["Sheet1", "Sheet2"],                  "exposed_type": "modin",              },          )          df_equals(dn.read()["Sheet1"], default_multi_sheet_data_frame["Sheet1"])          df_equals(dn.read()["Sheet2"], default_multi_sheet_data_frame["Sheet2"])            dn.append(content)            df_equals(              dn.read()["Sheet1"],              modin_pd.concat(                  [                      default_multi_sheet_data_frame["Sheet1"],                      modin_pd.DataFrame(content["Sheet1"], columns=["a", "b", "c"]),                  ]              ).reset_index(drop=True),          )          df_equals(              dn.read()["Sheet2"],              modin_pd.concat(                  [                      default_multi_sheet_data_frame["Sheet2"],                      modin_pd.DataFrame(content["Sheet2"], columns=["a", "b", "c"]),                  ]              ).reset_index(drop=True),          )        def test_filter_pandas_exposed_type_with_sheetname(self, excel_file):          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file, "sheet_name": "Sheet1", "exposed_type": "pandas"}          )          dn.write(              [                  {"foo": 1, "bar": 1},                  {"foo": 1, "bar": 2},                  {"foo": 1},                  {"foo": 2, "bar": 2},                  {"bar": 2},              ]          )            # Test datanode indexing and slicing          assert dn["foo"].equals(pd.Series([1, 1, 1, 2, None]))          assert dn["bar"].equals(pd.Series([1, 2, None, 2, 2]))          assert dn[:2].equals(pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))            # Test filter data          filtered_by_filter_method = dn.filter(("foo", 1, Operator.EQUAL))          filtered_by_indexing = dn[dn["foo"] == 1]          expected_data = pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}, {"foo": 1.0}])          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("foo", 1, Operator.NOT_EQUAL))          filtered_by_indexing = dn[dn["foo"] != 1]          expected_data = pd.DataFrame([{"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("bar", 2, Operator.EQUAL))          filtered_by_indexing = dn[dn["bar"] == 2]          expected_data = pd.DataFrame([{"foo": 1.0, "bar": 2.0}, {"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)          filtered_by_indexing = dn[(dn["bar"] == 1) | (dn["bar"] == 2)]          expected_data = pd.DataFrame(              [                  {"foo": 1.0, "bar": 1.0},                  {"foo": 1.0, "bar": 2.0},                  {"foo": 2.0, "bar": 2.0},                  {"bar": 2.0},              ]          )          assert_frame_equal(filtered_by_filter_method.reset_index(drop=True), expected_data)          assert_frame_equal(filtered_by_indexing.reset_index(drop=True), expected_data)        def test_filter_pandas_exposed_type_without_sheetname(self, excel_file):          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file, "exposed_type": "pandas"})          dn.write(              [                  {"foo": 1, "bar": 1},                  {"foo": 1, "bar": 2},                  {"foo": 1},                  {"foo": 2, "bar": 2},                  {"bar": 2},              ]          )            assert len(dn.filter(("foo", 1, Operator.EQUAL))["Sheet1"]) == 3          assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))["Sheet1"]) == 2          assert len(dn.filter(("bar", 2, Operator.EQUAL))["Sheet1"]) == 3          assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)["Sheet1"]) == 4            assert dn["Sheet1"]["foo"].equals(pd.Series([1, 1, 1, 2, None]))          assert dn["Sheet1"]["bar"].equals(pd.Series([1, 2, None, 2, 2]))          assert dn["Sheet1"][:2].equals(pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))        def test_filter_pandas_exposed_type_multisheet(self, excel_file):          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": excel_file, "sheet_name": ["sheet_1", "sheet_2"], "exposed_type": "pandas"},          )          dn.write(              {                  "sheet_1": pd.DataFrame(                      [                          {"foo": 1, "bar": 1},                          {"foo": 1, "bar": 2},                          {"foo": 1},                          {"foo": 2, "bar": 2},                          {"bar": 2},                      ]                  ),                  "sheet_2": pd.DataFrame(                      [                          {"foo": 1, "bar": 3},                          {"foo": 1, "bar": 4},                          {"foo": 1},                          {"foo": 2, "bar": 4},                          {"bar": 4},                      ]                  ),              }          )            assert len(dn.filter(("foo", 1, Operator.EQUAL))) == 2          assert len(dn.filter(("foo", 1, Operator.EQUAL))["sheet_1"]) == 3          assert len(dn.filter(("foo", 1, Operator.EQUAL))["sheet_2"]) == 3            assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))) == 2          assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))["sheet_1"]) == 2          assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))["sheet_2"]) == 2            assert len(dn.filter(("bar", 2, Operator.EQUAL))) == 2          assert len(dn.filter(("bar", 2, Operator.EQUAL))["sheet_1"]) == 3          assert len(dn.filter(("bar", 2, Operator.EQUAL))["sheet_2"]) == 0            assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)) == 2          assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)["sheet_1"]) == 4          assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)["sheet_2"]) == 0            assert dn["sheet_1"]["foo"].equals(pd.Series([1, 1, 1, 2, None]))          assert dn["sheet_2"]["foo"].equals(pd.Series([1, 1, 1, 2, None]))          assert dn["sheet_1"]["bar"].equals(pd.Series([1, 2, None, 2, 2]))          assert dn["sheet_2"]["bar"].equals(pd.Series([3, 4, None, 4, 4]))          assert dn["sheet_1"][:2].equals(pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))          assert dn["sheet_2"][:2].equals(pd.DataFrame([{"foo": 1.0, "bar": 3.0}, {"foo": 1.0, "bar": 4.0}]))        def test_filter_modin_exposed_type_with_sheetname(self, excel_file):          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file, "sheet_name": "Sheet1", "exposed_type": "modin"}          )          dn.write(              [                  {"foo": 1, "bar": 1},                  {"foo": 1, "bar": 2},                  {"foo": 1},                  {"foo": 2, "bar": 2},                  {"bar": 2},              ]          )            # Test datanode indexing and slicing          assert dn["foo"].equals(modin_pd.Series([1, 1, 1, 2, None]))          assert dn["bar"].equals(modin_pd.Series([1, 2, None, 2, 2]))          assert dn[:2].equals(modin_pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))            # Test filter data          filtered_by_filter_method = dn.filter(("foo", 1, Operator.EQUAL))          filtered_by_indexing = dn[dn["foo"] == 1]          expected_data = modin_pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}, {"foo": 1.0}])          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("foo", 1, Operator.NOT_EQUAL))          filtered_by_indexing = dn[dn["foo"] != 1]          expected_data = modin_pd.DataFrame([{"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter(("bar", 2, Operator.EQUAL))          filtered_by_indexing = dn[dn["bar"] == 2]          expected_data = modin_pd.DataFrame([{"foo": 1.0, "bar": 2.0}, {"foo": 2.0, "bar": 2.0}, {"bar": 2.0}])          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)            filtered_by_filter_method = dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)          filtered_by_indexing = dn[(dn["bar"] == 1) | (dn["bar"] == 2)]          expected_data = modin_pd.DataFrame(              [                  {"foo": 1.0, "bar": 1.0},                  {"foo": 1.0, "bar": 2.0},                  {"foo": 2.0, "bar": 2.0},                  {"bar": 2.0},              ]          )          df_equals(filtered_by_filter_method.reset_index(drop=True), expected_data)          df_equals(filtered_by_indexing.reset_index(drop=True), expected_data)        def test_filter_modin_exposed_type_without_sheetname(self, excel_file):          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file, "exposed_type": "modin"})          dn.write(              [                  {"foo": 1, "bar": 1},                  {"foo": 1, "bar": 2},                  {"foo": 1},                  {"foo": 2, "bar": 2},                  {"bar": 2},              ]          )            assert len(dn.filter(("foo", 1, Operator.EQUAL))["Sheet1"]) == 3          assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))["Sheet1"]) == 2          assert len(dn.filter(("bar", 2, Operator.EQUAL))["Sheet1"]) == 3          assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)["Sheet1"]) == 4            assert dn["Sheet1"]["foo"].equals(modin_pd.Series([1, 1, 1, 2, None]))          assert dn["Sheet1"]["bar"].equals(modin_pd.Series([1, 2, None, 2, 2]))          assert dn["Sheet1"][:2].equals(modin_pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))        def test_filter_modin_exposed_type_multisheet(self, excel_file):          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": excel_file, "sheet_name": ["sheet_1", "sheet_2"], "exposed_type": "modin"},          )          dn.write(              {                  "sheet_1": pd.DataFrame(                      [                          {"foo": 1, "bar": 1},                          {"foo": 1, "bar": 2},                          {"foo": 1},                          {"foo": 2, "bar": 2},                          {"bar": 2},                      ]                  ),                  "sheet_2": pd.DataFrame(                      [                          {"foo": 1, "bar": 3},                          {"foo": 1, "bar": 4},                          {"foo": 1},                          {"foo": 2, "bar": 4},                          {"bar": 4},                      ]                  ),              }          )            assert len(dn.filter(("foo", 1, Operator.EQUAL))) == 2          assert len(dn.filter(("foo", 1, Operator.EQUAL))["sheet_1"]) == 3          assert len(dn.filter(("foo", 1, Operator.EQUAL))["sheet_2"]) == 3            assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))) == 2          assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))["sheet_1"]) == 2          assert len(dn.filter(("foo", 1, Operator.NOT_EQUAL))["sheet_2"]) == 2            assert len(dn.filter(("bar", 2, Operator.EQUAL))) == 2          assert len(dn.filter(("bar", 2, Operator.EQUAL))["sheet_1"]) == 3          assert len(dn.filter(("bar", 2, Operator.EQUAL))["sheet_2"]) == 0            assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)) == 2          assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)["sheet_1"]) == 4          assert len(dn.filter([("bar", 1, Operator.EQUAL), ("bar", 2, Operator.EQUAL)], JoinOperator.OR)["sheet_2"]) == 0            assert dn["sheet_1"]["foo"].equals(modin_pd.Series([1, 1, 1, 2, None]))          assert dn["sheet_2"]["foo"].equals(modin_pd.Series([1, 1, 1, 2, None]))          assert dn["sheet_1"]["bar"].equals(modin_pd.Series([1, 2, None, 2, 2]))          assert dn["sheet_2"]["bar"].equals(modin_pd.Series([3, 4, None, 4, 4]))          assert dn["sheet_1"][:2].equals(modin_pd.DataFrame([{"foo": 1.0, "bar": 1.0}, {"foo": 1.0, "bar": 2.0}]))          assert dn["sheet_2"][:2].equals(modin_pd.DataFrame([{"foo": 1.0, "bar": 3.0}, {"foo": 1.0, "bar": 4.0}]))        def test_filter_numpy_exposed_type_with_sheetname(self, excel_file):          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"path": excel_file, "sheet_name": "Sheet1", "exposed_type": "numpy"}          )          dn.write(              [                  [1, 1],                  [1, 2],                  [1, 3],                  [2, 1],                  [2, 2],                  [2, 3],              ]          )            # Test datanode indexing and slicing          assert np.array_equal(dn[0], np.array([1, 1]))          assert np.array_equal(dn[1], np.array([1, 2]))          assert np.array_equal(dn[:3], np.array([[1, 1], [1, 2], [1, 3]]))          assert np.array_equal(dn[:, 0], np.array([1, 1, 1, 2, 2, 2]))          assert np.array_equal(dn[1:4, :1], np.array([[1], [1], [2]]))            # Test filter data          assert np.array_equal(dn.filter((0, 1, Operator.EQUAL)), np.array([[1, 1], [1, 2], [1, 3]]))          assert np.array_equal(dn[dn[:, 0] == 1], np.array([[1, 1], [1, 2], [1, 3]]))            assert np.array_equal(dn.filter((0, 1, Operator.NOT_EQUAL)), np.array([[2, 1], [2, 2], [2, 3]]))          assert np.array_equal(dn[dn[:, 0] != 1], np.array([[2, 1], [2, 2], [2, 3]]))            assert np.array_equal(dn.filter((1, 2, Operator.EQUAL)), np.array([[1, 2], [2, 2]]))          assert np.array_equal(dn[dn[:, 1] == 2], np.array([[1, 2], [2, 2]]))            assert np.array_equal(              dn.filter([(1, 1, Operator.EQUAL), (1, 2, Operator.EQUAL)], JoinOperator.OR),              np.array([[1, 1], [1, 2], [2, 1], [2, 2]]),          )          assert np.array_equal(dn[(dn[:, 1] == 1) | (dn[:, 1] == 2)], np.array([[1, 1], [1, 2], [2, 1], [2, 2]]))        def test_filter_numpy_exposed_type_without_sheetname(self, excel_file):          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": excel_file, "exposed_type": "numpy"})          dn.write(              [                  [1, 1],                  [1, 2],                  [1, 3],                  [2, 1],                  [2, 2],                  [2, 3],              ]          )            assert len(dn.filter((0, 1, Operator.EQUAL))["Sheet1"]) == 3          assert len(dn.filter((0, 1, Operator.NOT_EQUAL))["Sheet1"]) == 3          assert len(dn.filter((1, 2, Operator.EQUAL))["Sheet1"]) == 2          assert len(dn.filter([(0, 1, Operator.EQUAL), (1, 2, Operator.EQUAL)], JoinOperator.OR)["Sheet1"]) == 4            assert np.array_equal(dn["Sheet1"][0], np.array([1, 1]))          assert np.array_equal(dn["Sheet1"][1], np.array([1, 2]))          assert np.array_equal(dn["Sheet1"][:3], np.array([[1, 1], [1, 2], [1, 3]]))          assert np.array_equal(dn["Sheet1"][:, 0], np.array([1, 1, 1, 2, 2, 2]))          assert np.array_equal(dn["Sheet1"][1:4, :1], np.array([[1], [1], [2]]))        def test_filter_numpy_exposed_type_multisheet(self, excel_file):          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"path": excel_file, "sheet_name": ["sheet_1", "sheet_2"], "exposed_type": "numpy"},          )          dn.write(              {                  "sheet_1": pd.DataFrame(                      [                          [1, 1],                          [1, 2],                          [1, 3],                          [2, 1],                          [2, 2],                          [2, 3],                      ]                  ),                  "sheet_2": pd.DataFrame(                      [                          [1, 4],                          [1, 5],                          [1, 6],                          [2, 4],                          [2, 5],                          [2, 6],                      ]                  ),              }          )            assert len(dn.filter((0, 1, Operator.EQUAL))) == 2          assert len(dn.filter((0, 1, Operator.EQUAL))["sheet_1"]) == 3          assert len(dn.filter((0, 1, Operator.EQUAL))["sheet_2"]) == 3            assert len(dn.filter((0, 1, Operator.NOT_EQUAL))) == 2          assert len(dn.filter((0, 1, Operator.NOT_EQUAL))["sheet_1"]) == 3          assert len(dn.filter((0, 1, Operator.NOT_EQUAL))["sheet_2"]) == 3            assert len(dn.filter((1, 2, Operator.EQUAL))) == 2          assert len(dn.filter((1, 2, Operator.EQUAL))["sheet_1"]) == 2          assert len(dn.filter((1, 2, Operator.EQUAL))["sheet_2"]) == 0            assert len(dn.filter([(1, 1, Operator.EQUAL), (1, 2, Operator.EQUAL)], JoinOperator.OR)) == 2          assert len(dn.filter([(1, 1, Operator.EQUAL), (1, 2, Operator.EQUAL)], JoinOperator.OR)["sheet_1"]) == 4          assert len(dn.filter([(1, 1, Operator.EQUAL), (1, 2, Operator.EQUAL)], JoinOperator.OR)["sheet_2"]) == 0            assert np.array_equal(dn["sheet_1"][0], np.array([1, 1]))          assert np.array_equal(dn["sheet_2"][0], np.array([1, 4]))          assert np.array_equal(dn["sheet_1"][1], np.array([1, 2]))          assert np.array_equal(dn["sheet_2"][1], np.array([1, 5]))          assert np.array_equal(dn["sheet_1"][:3], np.array([[1, 1], [1, 2], [1, 3]]))          assert np.array_equal(dn["sheet_2"][:3], np.array([[1, 4], [1, 5], [1, 6]]))          assert np.array_equal(dn["sheet_1"][:, 0], np.array([1, 1, 1, 2, 2, 2]))          assert np.array_equal(dn["sheet_2"][:, 1], np.array([4, 5, 6, 4, 5, 6]))          assert np.array_equal(dn["sheet_1"][1:4, :1], np.array([[1], [1], [2]]))          assert np.array_equal(dn["sheet_2"][1:4, 1:2], np.array([[5], [6], [4]]))        def test_set_path(self):          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"default_path": "foo.xlsx"})          assert dn.path == "foo.xlsx"          dn.path = "bar.xlsx"          assert dn.path == "bar.xlsx"        @pytest.mark.parametrize(          ["properties", "exists"],          [              ({}, False),              ({"default_data": {"a": ["foo", "bar"]}}, True),          ],      )      def test_create_with_default_data(self, properties, exists):          dn = ExcelDataNode("foo", Scope.SCENARIO, DataNodeId("dn_id"), properties=properties)          assert os.path.exists(dn.path) is exists        def test_read_write_after_modify_path(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")          new_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/temp.xlsx")          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"default_path": path})          read_data = dn.read()          assert read_data is not None          dn.path = new_path          with pytest.raises(FileNotFoundError):              dn.read()          dn.write(read_data)          for sheet, df in dn.read().items():              assert np.array_equal(df.values, read_data[sheet].values)        def test_exposed_type_custom_class_after_modify_path(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")  # ["Sheet1", "Sheet2"]          new_path = os.path.join(              pathlib.Path(__file__).parent.resolve(), "data_sample/example_2.xlsx"          )  # ["Sheet1", "Sheet2", "Sheet3"]          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"default_path": path, "exposed_type": MyCustomObject1})          assert dn.exposed_type == MyCustomObject1          dn.read()          dn.path = new_path          dn.read()            dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"default_path": path, "exposed_type": MyCustomObject1, "sheet_name": ["Sheet4"]},          )          assert dn.exposed_type == MyCustomObject1          with pytest.raises(NonExistingExcelSheet):              dn.read()        def test_exposed_type_dict(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")  # ["Sheet1", "Sheet2"]          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={                  "default_path": path,                  "exposed_type": {                      "Sheet1": MyCustomObject1,                      "Sheet2": MyCustomObject2,                      "Sheet3": MyCustomObject1,                  },              },          )          data = dn.read()          assert isinstance(data, Dict)          assert isinstance(data["Sheet1"][0], MyCustomObject1)          assert isinstance(data["Sheet2"][0], MyCustomObject2)        def test_exposed_type_list(self):          path_1 = os.path.join(              pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx"          )  # ["Sheet1", "Sheet2"]          path_2 = os.path.join(              pathlib.Path(__file__).parent.resolve(), "data_sample/example_2.xlsx"          )  # ["Sheet1", "Sheet2", "Sheet3"]            dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"default_path": path_1, "exposed_type": [MyCustomObject1, MyCustomObject2]},          )          data = dn.read()          assert isinstance(data, Dict)          assert isinstance(data["Sheet1"][0], MyCustomObject1)          assert isinstance(data["Sheet2"][0], MyCustomObject2)            dn.path = path_2          with pytest.raises(ExposedTypeLengthMismatch):              dn.read()        def test_not_trying_to_read_sheet_names_when_exposed_type_is_set(self):          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"default_path": "notexistyet.xlsx", "exposed_type": MyCustomObject1}          )          assert dn.path == "notexistyet.xlsx"          assert dn.exposed_type == MyCustomObject1          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={"default_path": "notexistyet.xlsx", "exposed_type": [MyCustomObject1, MyCustomObject2]},          )          assert dn.path == "notexistyet.xlsx"          assert dn.exposed_type == [MyCustomObject1, MyCustomObject2]          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={                  "default_path": "notexistyet.xlsx",                  "exposed_type": {"Sheet1": MyCustomObject1, "Sheet2": MyCustomObject2},              },          )          assert dn.path == "notexistyet.xlsx"          assert dn.exposed_type == {"Sheet1": MyCustomObject1, "Sheet2": MyCustomObject2}        def test_exposed_type_default(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"default_path": path, "sheet_name": "Sheet1"})          assert dn.exposed_type == "pandas"          data = dn.read()          assert isinstance(data, pd.DataFrame)        def test_pandas_exposed_type(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")          dn = ExcelDataNode(              "foo", Scope.SCENARIO, properties={"default_path": path, "exposed_type": "pandas", "sheet_name": "Sheet1"}          )          assert dn.exposed_type == "pandas"          data = dn.read()          assert isinstance(data, pd.DataFrame)        def test_complex_exposed_type_dict(self):          # ["Sheet1", "Sheet2", "Sheet3", "Sheet4", "Sheet5"]          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example_4.xlsx")          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={                  "default_path": path,                  "exposed_type": {                      "Sheet1": MyCustomObject1,                      "Sheet2": "numpy",                      "Sheet3": "pandas",                  },                  "sheet_name": ["Sheet1", "Sheet2", "Sheet3", "Sheet4"],              },          )          data = dn.read()          assert isinstance(data, dict)          assert isinstance(data["Sheet1"], list)          assert isinstance(data["Sheet1"][0], MyCustomObject1)          assert isinstance(data["Sheet2"], np.ndarray)          assert isinstance(data["Sheet3"], pd.DataFrame)          assert isinstance(data["Sheet4"], pd.DataFrame)          assert data.get("Sheet5") is None        def test_complex_exposed_type_list(self):          # ["Sheet1", "Sheet2", "Sheet3", "Sheet4","Sheet5"]          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example_4.xlsx")          dn = ExcelDataNode(              "foo",              Scope.SCENARIO,              properties={                  "default_path": path,                  "exposed_type": [MyCustomObject1, "numpy", "pandas"],                  "sheet_name": ["Sheet1", "Sheet2", "Sheet3"],              },          )          data = dn.read()          assert isinstance(data, dict)          assert isinstance(data["Sheet1"], list)          assert isinstance(data["Sheet1"][0], MyCustomObject1)          assert isinstance(data["Sheet2"], np.ndarray)          assert isinstance(data["Sheet3"], pd.DataFrame)        def test_invalid_exposed_type(self):          path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")          with pytest.raises(InvalidExposedType):              ExcelDataNode(                  "foo",                  Scope.SCENARIO,                  properties={"default_path": path, "exposed_type": "invalid", "sheet_name": "Sheet1"},              )            with pytest.raises(InvalidExposedType):              ExcelDataNode(                  "foo",                  Scope.SCENARIO,                  properties={                      "default_path": path,                      "exposed_type": ["numpy", "invalid", "pandas"],                      "sheet_name": "Sheet1",                  },              )            with pytest.raises(InvalidExposedType):              ExcelDataNode(                  "foo",                  Scope.SCENARIO,                  properties={                      "default_path": path,                      "exposed_type": {"Sheet1": "pandas", "Sheet2": "invalid"},                      "sheet_name": "Sheet1",                  },              )        def test_get_system_modified_date_instead_of_last_edit_date(self, tmpdir_factory):          temp_file_path = str(tmpdir_factory.mktemp("data").join("temp.xlsx"))          pd.DataFrame([]).to_excel(temp_file_path)          dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": temp_file_path, "exposed_type": "pandas"})            dn.write(pd.DataFrame([1, 2, 3]))          previous_edit_date = dn.last_edit_date            sleep(0.1)            pd.DataFrame([4, 5, 6]).to_excel(temp_file_path)          new_edit_date = datetime.fromtimestamp(os.path.getmtime(temp_file_path))            assert previous_edit_date < dn.last_edit_date          assert new_edit_date == dn.last_edit_date            sleep(0.1)            dn.write(pd.DataFrame([7, 8, 9]))          assert new_edit_date < dn.last_edit_date          os.unlink(temp_file_path)


import os  import pathlib    import pytest    from src.taipy.core._version._version_manager import _VersionManager  from src.taipy.core.config.data_node_config import DataNodeConfig  from src.taipy.core.data._data_manager import _DataManager  from src.taipy.core.data._data_manager_factory import _DataManagerFactory  from src.taipy.core.data.csv import CSVDataNode  from src.taipy.core.data.data_node_id import DataNodeId  from src.taipy.core.data.in_memory import InMemoryDataNode  from src.taipy.core.exceptions.exceptions import InvalidDataNodeType, ModelNotFound  from taipy.config.common.scope import Scope  from taipy.config.config import Config      def file_exists(file_path: str) -> bool:      return os.path.exists(file_path)      def init_managers():      _DataManagerFactory._build_manager()._delete_all()      class TestDataManager:      def test_create_data_node_and_modify_properties_does_not_modify_config(self, init_sql_repo):          init_managers()            dn_config = Config.configure_data_node(id="name", foo="bar")          dn = _DataManager._create_and_set(dn_config, None, None)          assert dn_config.properties.get("foo") == "bar"          assert dn_config.properties.get("baz") is None            dn.properties["baz"] = "qux"          _DataManager._set(dn)          assert dn_config.properties.get("foo") == "bar"          assert dn_config.properties.get("baz") is None          assert dn.properties.get("foo") == "bar"          assert dn.properties.get("baz") == "qux"        def test_create_raises_exception_with_wrong_type(self, init_sql_repo):          init_managers()            wrong_type_dn_config = DataNodeConfig(id="foo", storage_type="bar", scope=DataNodeConfig._DEFAULT_SCOPE)          with pytest.raises(InvalidDataNodeType):              _DataManager._create_and_set(wrong_type_dn_config, None, None)        def test_create_from_same_config_generates_new_data_node_and_new_id(self, init_sql_repo):          init_managers()            dn_config = Config.configure_data_node(id="foo", storage_type="in_memory")          dn = _DataManager._create_and_set(dn_config, None, None)          dn_2 = _DataManager._create_and_set(dn_config, None, None)          assert dn_2.id != dn.id        def test_create_uses_overridden_attributes_in_config_file(self, init_sql_repo):          init_managers()            Config.override(os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/config.toml"))            csv_dn_cfg = Config.configure_data_node(id="foo", storage_type="csv", path="bar", has_header=True)          csv_dn = _DataManager._create_and_set(csv_dn_cfg, None, None)          assert csv_dn.config_id == "foo"          assert isinstance(csv_dn, CSVDataNode)          assert csv_dn._path == "path_from_config_file"          assert csv_dn.has_header            csv_dn_cfg = Config.configure_data_node(id="baz", storage_type="csv", path="bar", has_header=True)          csv_dn = _DataManager._create_and_set(csv_dn_cfg, None, None)          assert csv_dn.config_id == "baz"          assert isinstance(csv_dn, CSVDataNode)          assert csv_dn._path == "bar"          assert csv_dn.has_header        def test_get_if_not_exists(self, init_sql_repo):          init_managers()            with pytest.raises(ModelNotFound):              _DataManager._repository._load("test_data_node_2")        def test_get_all(self, init_sql_repo):          init_managers()            _DataManager._delete_all()          assert len(_DataManager._get_all()) == 0          dn_config_1 = Config.configure_data_node(id="foo", storage_type="in_memory")          _DataManager._create_and_set(dn_config_1, None, None)          assert len(_DataManager._get_all()) == 1          dn_config_2 = Config.configure_data_node(id="baz", storage_type="in_memory")          _DataManager._create_and_set(dn_config_2, None, None)          _DataManager._create_and_set(dn_config_2, None, None)          assert len(_DataManager._get_all()) == 3          assert len([dn for dn in _DataManager._get_all() if dn.config_id == "foo"]) == 1          assert len([dn for dn in _DataManager._get_all() if dn.config_id == "baz"]) == 2        def test_get_all_on_multiple_versions_environment(self, init_sql_repo):          init_managers()            # Create 5 data nodes with 2 versions each          # Only version 1.0 has the data node with config_id = "config_id_1"          # Only version 2.0 has the data node with config_id = "config_id_6"          for version in range(1, 3):              for i in range(5):                  _DataManager._set(                      InMemoryDataNode(                          f"config_id_{i+version}",                          Scope.SCENARIO,                          id=DataNodeId(f"id{i}_v{version}"),                          version=f"{version}.0",                      )                  )            _VersionManager._set_experiment_version("1.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_1"}])) == 1          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_6"}])) == 0            _VersionManager._set_development_version("1.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_1"}])) == 1          assert len(_DataManager._get_all_by(filters=[{"version": "1.0", "config_id": "config_id_6"}])) == 0            _VersionManager._set_experiment_version("2.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_1"}])) == 0          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_6"}])) == 1            _VersionManager._set_development_version("2.0")          assert len(_DataManager._get_all()) == 5          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_1"}])) == 0          assert len(_DataManager._get_all_by(filters=[{"version": "2.0", "config_id": "config_id_6"}])) == 1        def test_set(self, init_sql_repo):          init_managers()            dn = InMemoryDataNode(              "config_id",              Scope.SCENARIO,              id=DataNodeId("id"),              owner_id=None,              parent_ids={"task_id_1"},              last_edit_date=None,              edits=[],              edit_in_progress=False,              properties={"foo": "bar"},          )          assert len(_DataManager._get_all()) == 0          assert not _DataManager._exists(dn.id)          _DataManager._set(dn)          assert len(_DataManager._get_all()) == 1          assert _DataManager._exists(dn.id)            # changing data node attribute          dn.config_id = "foo"          assert dn.config_id == "foo"          _DataManager._set(dn)          assert len(_DataManager._get_all()) == 1          assert dn.config_id == "foo"          assert _DataManager._get(dn.id).config_id == "foo"        def test_delete(self, init_sql_repo):          init_managers()          _DataManager._delete_all()            dn_1 = InMemoryDataNode("config_id", Scope.SCENARIO, id="id_1")          dn_2 = InMemoryDataNode("config_id", Scope.SCENARIO, id="id_2")          dn_3 = InMemoryDataNode("config_id", Scope.SCENARIO, id="id_3")          assert len(_DataManager._get_all()) == 0          _DataManager._set(dn_1)          _DataManager._set(dn_2)          _DataManager._set(dn_3)          assert len(_DataManager._get_all()) == 3          assert all(_DataManager._exists(dn.id) for dn in [dn_1, dn_2, dn_3])          _DataManager._delete(dn_1.id)          assert len(_DataManager._get_all()) == 2          assert _DataManager._get(dn_2.id).id == dn_2.id          assert _DataManager._get(dn_3.id).id == dn_3.id          assert _DataManager._get(dn_1.id) is None          assert all(_DataManager._exists(dn.id) for dn in [dn_2, dn_3])          assert not _DataManager._exists(dn_1.id)          _DataManager._delete_all()          assert len(_DataManager._get_all()) == 0          assert not any(_DataManager._exists(dn.id) for dn in [dn_2, dn_3])        def test_get_or_create(self, init_sql_repo):          def _get_or_create_dn(config, *args):              return _DataManager._bulk_get_or_create([config], *args)[config]            init_managers()            global_dn_config = Config.configure_data_node(              id="test_data_node", storage_type="in_memory", scope=Scope.GLOBAL, data="In memory Data Node"          )          cycle_dn_config = Config.configure_data_node(              id="test_data_node1", storage_type="in_memory", scope=Scope.CYCLE, data="In memory scenario"          )          scenario_dn_config = Config.configure_data_node(              id="test_data_node2", storage_type="in_memory", scope=Scope.SCENARIO, data="In memory scenario"          )            _DataManager._delete_all()            assert len(_DataManager._get_all()) == 0          global_dn = _get_or_create_dn(global_dn_config, None, None)          assert len(_DataManager._get_all()) == 1          global_dn_bis = _get_or_create_dn(global_dn_config, None)          assert len(_DataManager._get_all()) == 1          assert global_dn.id == global_dn_bis.id            scenario_dn = _get_or_create_dn(scenario_dn_config, None, "scenario_id")          assert len(_DataManager._get_all()) == 2          scenario_dn_bis = _get_or_create_dn(scenario_dn_config, None, "scenario_id")          assert len(_DataManager._get_all()) == 2          assert scenario_dn.id == scenario_dn_bis.id          scenario_dn_ter = _get_or_create_dn(scenario_dn_config, None, "scenario_id")          assert len(_DataManager._get_all()) == 2          assert scenario_dn.id == scenario_dn_bis.id          assert scenario_dn_bis.id == scenario_dn_ter.id          scenario_dn_quater = _get_or_create_dn(scenario_dn_config, None, "scenario_id_2")          assert len(_DataManager._get_all()) == 3          assert scenario_dn.id == scenario_dn_bis.id          assert scenario_dn_bis.id == scenario_dn_ter.id          assert scenario_dn_ter.id != scenario_dn_quater.id            assert len(_DataManager._get_all()) == 3          cycle_dn = _get_or_create_dn(cycle_dn_config, "cycle_id", None)          assert len(_DataManager._get_all()) == 4          cycle_dn_1 = _get_or_create_dn(cycle_dn_config, "cycle_id", None)          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_1.id          cycle_dn_2 = _get_or_create_dn(cycle_dn_config, "cycle_id", "scenario_id")          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_2.id          cycle_dn_3 = _get_or_create_dn(cycle_dn_config, "cycle_id", None)          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_3.id          cycle_dn_4 = _get_or_create_dn(cycle_dn_config, "cycle_id", "scenario_id")          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_4.id          cycle_dn_5 = _get_or_create_dn(cycle_dn_config, "cycle_id", "scenario_id_2")          assert len(_DataManager._get_all()) == 4          assert cycle_dn.id == cycle_dn_5.id            assert cycle_dn_1.id == cycle_dn_2.id          assert cycle_dn_2.id == cycle_dn_3.id          assert cycle_dn_3.id == cycle_dn_4.id          assert cycle_dn_4.id == cycle_dn_5.id        def test_get_data_nodes_by_config_id(self, init_sql_repo):          init_managers()            dn_config_1 = Config.configure_data_node("dn_1", scope=Scope.SCENARIO)          dn_config_2 = Config.configure_data_node("dn_2", scope=Scope.SCENARIO)          dn_config_3 = Config.configure_data_node("dn_3", scope=Scope.SCENARIO)            dn_1_1 = _DataManager._create_and_set(dn_config_1, None, None)          dn_1_2 = _DataManager._create_and_set(dn_config_1, None, None)          dn_1_3 = _DataManager._create_and_set(dn_config_1, None, None)          assert len(_DataManager._get_all()) == 3            dn_2_1 = _DataManager._create_and_set(dn_config_2, None, None)          dn_2_2 = _DataManager._create_and_set(dn_config_2, None, None)          assert len(_DataManager._get_all()) == 5            dn_3_1 = _DataManager._create_and_set(dn_config_3, None, None)          assert len(_DataManager._get_all()) == 6            dn_1_datanodes = _DataManager._get_by_config_id(dn_config_1.id)          assert len(dn_1_datanodes) == 3          assert sorted([dn_1_1.id, dn_1_2.id, dn_1_3.id]) == sorted([sequence.id for sequence in dn_1_datanodes])            dn_2_datanodes = _DataManager._get_by_config_id(dn_config_2.id)          assert len(dn_2_datanodes) == 2          assert sorted([dn_2_1.id, dn_2_2.id]) == sorted([sequence.id for sequence in dn_2_datanodes])            dn_3_datanodes = _DataManager._get_by_config_id(dn_config_3.id)          assert len(dn_3_datanodes) == 1          assert sorted([dn_3_1.id]) == sorted([sequence.id for sequence in dn_3_datanodes])        def test_get_data_nodes_by_config_id_in_multiple_versions_environment(self, init_sql_repo):          init_managers()            dn_config_1 = Config.configure_data_node("dn_1", scope=Scope.SCENARIO)          dn_config_2 = Config.configure_data_node("dn_2", scope=Scope.SCENARIO)            _VersionManager._set_experiment_version("1.0")          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_2, None, None)          _DataManager._create_and_set(dn_config_2, None, None)            assert len(_DataManager._get_by_config_id(dn_config_1.id)) == 3          assert len(_DataManager._get_by_config_id(dn_config_2.id)) == 2            _VersionManager._set_experiment_version("2.0")          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_1, None, None)          _DataManager._create_and_set(dn_config_2, None, None)          _DataManager._create_and_set(dn_config_2, None, None)            assert len(_DataManager._get_by_config_id(dn_config_1.id)) == 3          assert len(_DataManager._get_by_config_id(dn_config_2.id)) == 2


import os    import pytest    from src.taipy.core.data._data_manager_factory import _DataManagerFactory  from src.taipy.core.exceptions import ModelNotFound  from src.taipy.core.job._job_manager_factory import _JobManagerFactory  from src.taipy.core.submission._submission_manager_factory import _SubmissionManagerFactory  from src.taipy.core.submission.submission import Submission  from src.taipy.core.task._task_manager_factory import _TaskManagerFactory  from src.taipy.core.task.task import Task  from taipy.config.config import Config  from tests.conftest import init_sql_repo      def configure_fs_repo():      Config.configure_core(repository_type="default")      def configure_sql_repo():      init_sql_repo      class TestSubmissionRepository:      @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_save_and_load(self, data_node, job, configure_repo):          configure_repo()            _DataManagerFactory._build_manager()._repository._save(data_node)          task = Task("task_config_id", {}, print, [data_node], [data_node])          _TaskManagerFactory._build_manager()._repository._save(task)          job._task = task          _JobManagerFactory._build_manager()._repository._save(job)            submission = Submission(task.id)          submission_repository = _SubmissionManagerFactory._build_manager()._repository          submission_repository._save(submission)          submission.jobs = [job]            obj = submission_repository._load(submission.id)          assert isinstance(obj, Submission)        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_exists(self, configure_repo):          configure_repo()            submission = Submission("entity_id")          submission_repository = _SubmissionManagerFactory._build_manager()._repository          submission_repository._save(submission)            assert submission_repository._exists(submission.id)          assert not submission_repository._exists("not-existed-submission")        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_load_all(self, configure_repo):          configure_repo()            repository = _SubmissionManagerFactory._build_manager()._repository          submission = Submission("entity_id")          for i in range(10):              submission.id = f"submission-{i}"              repository._save(submission)          submissions = repository._load_all()            assert len(submissions) == 10        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_delete(self, configure_repo):          configure_repo()            repository = _SubmissionManagerFactory._build_manager()._repository            submission = Submission("entity_id")          repository._save(submission)            repository._delete(submission.id)            with pytest.raises(ModelNotFound):              repository._load(submission.id)        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_delete_all(self, configure_repo):          configure_repo()            submission_repository = _SubmissionManagerFactory._build_manager()._repository          submission = Submission("entity_id")            for i in range(10):              submission.id = f"submission-{i}"              submission_repository._save(submission)            assert len(submission_repository._load_all()) == 10            submission_repository._delete_all()            assert len(submission_repository._load_all()) == 0        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_delete_many(self, configure_repo):          configure_repo()            submission = Submission("entity_id")          submission_repository = _SubmissionManagerFactory._build_manager()._repository            for i in range(10):              submission.id = f"submission-{i}"              submission_repository._save(submission)            objs = submission_repository._load_all()          assert len(objs) == 10          ids = [x.id for x in objs[:3]]          submission_repository._delete_many(ids)            assert len(submission_repository._load_all()) == 7        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_delete_by(self, configure_repo):          configure_repo()            # Create 5 entities with version 1.0 and 5 entities with version 2.0          submission_repository = _SubmissionManagerFactory._build_manager()._repository          submission = Submission("entity_id")            for i in range(10):              submission.id = f"submission-{i}"              submission._version = f"{(i+1) // 5}.0"              submission_repository._save(submission)            objs = submission_repository._load_all()          assert len(objs) == 10          submission_repository._delete_by("version", "1.0")            assert len(submission_repository._load_all()) == 5        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_search(self, configure_repo):          configure_repo()            submission_repository = _SubmissionManagerFactory._build_manager()._repository          submission = Submission("entity_id", version="random_version_number")          for i in range(10):              submission.id = f"submission-{i}"              submission_repository._save(submission)            assert len(submission_repository._load_all()) == 10            objs = submission_repository._search("id", "submission-2")          assert len(objs) == 1          assert isinstance(objs[0], Submission)            objs = submission_repository._search("id", "submission-2", filters=[{"version": "random_version_number"}])          assert len(objs) == 1          assert isinstance(objs[0], Submission)            assert submission_repository._search("id", "submission-2", filters=[{"version": "non_existed_version"}]) == []        @pytest.mark.parametrize("configure_repo", [configure_fs_repo, configure_sql_repo])      def test_export(self, tmpdir, configure_repo):          configure_repo()            repository = _SubmissionManagerFactory._build_manager()._repository          submission = Submission("entity_id")          repository._save(submission)            repository._export(submission.id, tmpdir.strpath)          dir_path = (              repository.dir_path              if Config.core.repository_type == "default"              else os.path.join(tmpdir.strpath, "submission")          )            assert os.path.exists(os.path.join(dir_path, f"{submission.id}.json"))


#  # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with  # the License. You may obtain a copy of the License at  #  #        http://www.apache.org/licenses/LICENSE-2.0  #  # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on  # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the  # specific language governing permissions and limitations under the License.