from collections.abc import Hashable  from functools import reduce  from itertools import chain  from operator import and_, or_  from typing import Dict, Iterable, List, Tuple, Union    import modin.pandas as modin_pd  import numpy as np  import pandas as pd  from pandas.core.common import is_bool_indexer    from .operator import JoinOperator, Operator      class _FilterDataNode:      @staticmethod      def __is_pandas_object(data) -> bool:          return isinstance(data, (pd.DataFrame, modin_pd.DataFrame)) or isinstance(data, (pd.Series, modin_pd.DataFrame))        @staticmethod      def __is_multi_sheet_excel(data) -> bool:          if isinstance(data, Dict):              has_df_children = all([isinstance(e, (pd.DataFrame, modin_pd.DataFrame)) for e in data.values()])              has_list_children = all([isinstance(e, List) for e in data.values()])              has_np_array_children = all([isinstance(e, np.ndarray) for e in data.values()])              return has_df_children or has_list_children or has_np_array_children          return False        @staticmethod      def __is_list_of_dict(data) -> bool:          return all(isinstance(x, Dict) for x in data)        @staticmethod      def _filter_by_key(data, key):          if isinstance(key, int):              return _FilterDataNode.__getitem_int(data, key)            if isinstance(key, slice) or (isinstance(key, tuple) and any(isinstance(e, slice) for e in key)):              return _FilterDataNode.__getitem_slice(data, key)            if isinstance(key, Hashable):              return _FilterDataNode.__getitem_hashable(data, key)            if isinstance(key, (pd.DataFrame, modin_pd.DataFrame)):              return _FilterDataNode.__getitem_dataframe(data, key)            if is_bool_indexer(key):              return _FilterDataNode.__getitem_bool_indexer(data, key)            if isinstance(key, Iterable):              return _FilterDataNode.__getitem_iterable(data, key)            return None        @staticmethod      def __getitem_int(data, key):          return data[key]        @staticmethod      def __getitem_hashable(data, key):          if _FilterDataNode.__is_pandas_object(data) or _FilterDataNode.__is_multi_sheet_excel(data):              return data.get(key)          return [getattr(entry, key, None) for entry in data]        @staticmethod      def __getitem_slice(data, key):          return data[key]        @staticmethod      def __getitem_dataframe(data, key: Union[pd.DataFrame, modin_pd.DataFrame]):          if _FilterDataNode.__is_pandas_object(data):              return data[key]          if _FilterDataNode.__is_list_of_dict(data):              filtered_data = list()              for i, row in key.iterrows():                  filtered_row = dict()                  for col in row.index:                      filtered_row[col] = data[i][col] if row[col] else None                  filtered_data.append(filtered_row)              return filtered_data          return None        @staticmethod      def __getitem_bool_indexer(data, key):          if _FilterDataNode.__is_pandas_object(data):              return data[key]          return [e for i, e in enumerate(data) if key[i]]        @staticmethod      def __getitem_iterable(data, keys):          if _FilterDataNode.__is_pandas_object(data):              return data[keys]          filtered_data = []          for entry in data:              filtered_data.append({k: getattr(entry, k) for k in keys if hasattr(entry, k)})          return filtered_data        @staticmethod      def _filter(data, operators: Union[List, Tuple], join_operator=JoinOperator.AND):          if len(operators) == 0:              return data            if isinstance(data, Dict):              return {k: _FilterDataNode._filter(v, operators, join_operator) for k, v in data.items()}            if not ((isinstance(operators[0], list)) or (isinstance(operators[0], tuple))):              if isinstance(data, (pd.DataFrame, modin_pd.DataFrame)):                  return _FilterDataNode.__filter_dataframe_per_key_value(data, operators[0], operators[1], operators[2])              if isinstance(data, np.ndarray):                  list_operators = [operators]                  return _FilterDataNode.__filter_numpy_array(data, list_operators)              if isinstance(data, List):                  return _FilterDataNode.__filter_list_per_key_value(data, operators[0], operators[1], operators[2])          else:              if isinstance(data, (pd.DataFrame, modin_pd.DataFrame)):                  return _FilterDataNode.__filter_dataframe(data, operators, join_operator=join_operator)              if isinstance(data, np.ndarray):                  return _FilterDataNode.__filter_numpy_array(data, operators, join_operator=join_operator)              if isinstance(data, List):                  return _FilterDataNode.__filter_list(data, operators, join_operator=join_operator)          raise NotImplementedError        @staticmethod      def __filter_dataframe(          df_data: Union[pd.DataFrame, modin_pd.DataFrame], operators: Union[List, Tuple], join_operator=JoinOperator.AND      ):          filtered_df_data = []          if join_operator == JoinOperator.AND:              how = "inner"          elif join_operator == JoinOperator.OR:              how = "outer"          else:              return NotImplementedError          for key, value, operator in operators:              filtered_df_data.append(_FilterDataNode.__filter_dataframe_per_key_value(df_data, key, value, operator))            if isinstance(df_data, modin_pd.DataFrame):              if filtered_df_data:                  return _FilterDataNode.__modin_dataframe_merge(filtered_df_data, how)              return modin_pd.DataFrame()            return _FilterDataNode.__dataframe_merge(filtered_df_data, how) if filtered_df_data else pd.DataFrame()        @staticmethod      def __filter_dataframe_per_key_value(          df_data: Union[pd.DataFrame, modin_pd.DataFrame], key: str, value, operator: Operator      ):          df_by_col = df_data[key]          if operator == Operator.EQUAL:              df_by_col = df_by_col == value          if operator == Operator.NOT_EQUAL:              df_by_col = df_by_col != value          if operator == Operator.LESS_THAN:              df_by_col = df_by_col < value          if operator == Operator.LESS_OR_EQUAL:              df_by_col = df_by_col <= value          if operator == Operator.GREATER_THAN:              df_by_col = df_by_col > value          if operator == Operator.GREATER_OR_EQUAL:              df_by_col = df_by_col >= value          return df_data[df_by_col]        @staticmethod      def __dataframe_merge(df_list: List, how="inner"):          return reduce(lambda df1, df2: pd.merge(df1, df2, how=how), df_list)        @staticmethod      def __modin_dataframe_merge(df_list: List, how="inner"):          return reduce(lambda df1, df2: modin_pd.merge(df1, df2, how=how), df_list)        @staticmethod      def __filter_numpy_array(data: np.ndarray, operators: Union[List, Tuple], join_operator=JoinOperator.AND):          conditions = []          for key, value, operator in operators:              conditions.append(_FilterDataNode.__get_filter_condition_per_key_value(data, key, value, operator))            if join_operator == JoinOperator.AND:              join_conditions = reduce(and_, conditions)          elif join_operator == JoinOperator.OR:              join_conditions = reduce(or_, conditions)          else:              return NotImplementedError            return data[join_conditions]        @staticmethod      def __get_filter_condition_per_key_value(array_data: np.ndarray, key, value, operator: Operator):          if not isinstance(key, int):              key = int(key)            if operator == Operator.EQUAL:              return array_data[:, key] == value          if operator == Operator.NOT_EQUAL:              return array_data[:, key] != value          if operator == Operator.LESS_THAN:              return array_data[:, key] < value          if operator == Operator.LESS_OR_EQUAL:              return array_data[:, key] <= value          if operator == Operator.GREATER_THAN:              return array_data[:, key] > value          if operator == Operator.GREATER_OR_EQUAL:              return array_data[:, key] >= value            return NotImplementedError        @staticmethod      def __filter_list(list_data: List, operators: Union[List, Tuple], join_operator=JoinOperator.AND):          filtered_list_data = []          for key, value, operator in operators:              filtered_list_data.append(_FilterDataNode.__filter_list_per_key_value(list_data, key, value, operator))          if len(filtered_list_data) == 0:              return filtered_list_data          if join_operator == JoinOperator.AND:              return _FilterDataNode.__list_intersect(filtered_list_data)          elif join_operator == JoinOperator.OR:              merged_list = list(chain.from_iterable(filtered_list_data))              if all(isinstance(e, Dict) for e in merged_list):                  return list({frozenset(item.items()) for item in merged_list})              return list(set(merged_list))          else:              return NotImplementedError        @staticmethod      def __filter_list_per_key_value(list_data: List, key: str, value, operator: Operator):          filtered_list = []          for row in list_data:              if isinstance(row, Dict):                  row_value = row.get(key, None)              else:                  row_value = getattr(row, key, None)                if operator == Operator.EQUAL and row_value == value:                  filtered_list.append(row)              if operator == Operator.NOT_EQUAL and row_value != value:                  filtered_list.append(row)              if operator == Operator.LESS_THAN and row_value < value:                  filtered_list.append(row)              if operator == Operator.LESS_OR_EQUAL and row_value <= value:                  filtered_list.append(row)              if operator == Operator.GREATER_THAN and row_value > value:                  filtered_list.append(row)              if operator == Operator.GREATER_OR_EQUAL and row_value >= value:                  filtered_list.append(row)          return filtered_list        @staticmethod      def __list_intersect(list_data):          return list(set(list_data.pop()).intersection(*map(set, list_data)))


from datetime import datetime, timedelta  from typing import Any, Dict, List, Optional, Set    from taipy.config.common.scope import Scope    from .._version._version_manager_factory import _VersionManagerFactory  from ..exceptions.exceptions import MissingReadFunction, MissingRequiredProperty, MissingWriteFunction  from .data_node import DataNode  from .data_node_id import DataNodeId, Edit      class GenericDataNode(DataNode):      """Generic Data Node that uses custom read and write functions.        The read and write function for this data node type can be implemented is Python.        Attributes:          config_id (str): Identifier of the data node configuration. It must be a valid Python              identifier.          scope (Scope^): The scope of this data node.          id (str): The unique identifier of the data node.          owner_id (str): The identifier of the owner (sequence_id, scenario_id, cycle_id) or              `None`.          parent_ids (Optional[Set[str]]): The identifiers of the parent tasks or `None`.          last_edit_date (datetime): The date and time of the last modification.          edits (List[Edit^]): The ordered list of edits for that job.          version (str): The string indicates the application version of the data node to instantiate. If not provided,              the current version is used.          validity_period (Optional[timedelta]): The duration implemented as a timedelta since the last edit date for              which the data node can be considered up-to-date. Once the validity period has passed, the data node is              considered stale and relevant tasks will run even if they are skippable (see the              [Task management page](../core/entities/task-mgt.md) for more details).              If _validity_period_ is set to `None`, the data node is always up-to-date.          edit_in_progress (bool): True if a task computing the data node has been submitted              and not completed yet. False otherwise.          editor_id (Optional[str]): The identifier of the user who is currently editing the data node.          editor_expiration_date (Optional[datetime]): The expiration date of the editor lock.          properties (dict[str, Any]): A dictionary of additional properties. Note that the              _properties_ parameter must at least contain an entry for either _"read_fct"_ or              _"write_fct"_ representing the read and write functions.              Entries for _"read_fct_args"_ and _"write_fct_args"_ respectively represent              potential parameters for the _"read_fct"_ and _"write_fct"_ functions.      """        __STORAGE_TYPE = "generic"      _OPTIONAL_READ_FUNCTION_PROPERTY = "read_fct"      __READ_FUNCTION_ARGS_PROPERTY = "read_fct_args"      _OPTIONAL_WRITE_FUNCTION_PROPERTY = "write_fct"      __WRITE_FUNCTION_ARGS_PROPERTY = "write_fct_args"      _REQUIRED_PROPERTIES: List[str] = []      _REQUIRED_AT_LEAST_ONE_PROPERTY: List[str] = [_OPTIONAL_READ_FUNCTION_PROPERTY, _OPTIONAL_WRITE_FUNCTION_PROPERTY]        def __init__(          self,          config_id: str,          scope: Scope,          id: Optional[DataNodeId] = None,          owner_id: Optional[str] = None,          parent_ids: Optional[Set[str]] = None,          last_edit_date: Optional[datetime] = None,          edits: List[Edit] = None,          version: str = None,          validity_period: Optional[timedelta] = None,          edit_in_progress: bool = False,          editor_id: Optional[str] = None,          editor_expiration_date: Optional[datetime] = None,          properties: Dict = None,      ):          if properties is None:              properties = {}          if missing := set(self._REQUIRED_PROPERTIES) - set(properties.keys()):              raise MissingRequiredProperty(                  f"The following properties " f"{', '.join(x for x in missing)} were not informed and are required."              )            missing_optional_fcts = set(self._REQUIRED_AT_LEAST_ONE_PROPERTY) - set(properties.keys())          if len(missing_optional_fcts) == len(self._REQUIRED_AT_LEAST_ONE_PROPERTY):              raise MissingRequiredProperty(                  f"None of the following properties "                  f"{', '.join(x for x in missing)} were informed and at least one must be populated."              )          for missing_optional_fct in missing_optional_fcts:              properties[missing_optional_fct] = None            super().__init__(              config_id,              scope,              id,              owner_id,              parent_ids,              last_edit_date,              edits,              version or _VersionManagerFactory._build_manager()._get_latest_version(),              validity_period,              edit_in_progress,              editor_id,              editor_expiration_date,              **properties,          )          if not self._last_edit_date:              self._last_edit_date = datetime.now()            self._TAIPY_PROPERTIES.update(              {                  self.__READ_FUNCTION_ARGS_PROPERTY,                  self.__WRITE_FUNCTION_ARGS_PROPERTY,                  self._OPTIONAL_READ_FUNCTION_PROPERTY,                  self._OPTIONAL_WRITE_FUNCTION_PROPERTY,              }          )        @classmethod      def storage_type(cls) -> str:          return cls.__STORAGE_TYPE        def _read(self):          if read_fct := self.properties[self._OPTIONAL_READ_FUNCTION_PROPERTY]:              if read_fct_args := self.properties.get(self.__READ_FUNCTION_ARGS_PROPERTY, None):                  if not isinstance(read_fct_args, list):                      return read_fct(*[read_fct_args])                  return read_fct(*read_fct_args)              return read_fct()          raise MissingReadFunction(f"The read function is not defined in data node config {self.config_id}.")        def _write(self, data: Any):          if write_fct := self.properties[self._OPTIONAL_WRITE_FUNCTION_PROPERTY]:              if write_fct_args := self.properties.get(self.__WRITE_FUNCTION_ARGS_PROPERTY, None):                  if not isinstance(write_fct_args, list):                      return write_fct(data, *[write_fct_args])                  return write_fct(data, *write_fct_args)              return write_fct(data)          raise MissingWriteFunction(f"The write function is not defined in data node config {self.config_id}.")


from ..exceptions.exceptions import InvalidExposedType      class _AbstractTabularDataNode(object):      """Abstract base class for tabular data node implementations (CSVDataNode, ParquetDataNode, ExcelDataNode,      SQLTableDataNode and SQLDataNode) that are tabular representable."""        @staticmethod      def _check_exposed_type(exposed_type, valid_string_exposed_types):          if isinstance(exposed_type, str) and exposed_type not in valid_string_exposed_types:              raise InvalidExposedType(                  f"Invalid string exposed type {exposed_type}. Supported values are "                  f"{', '.join(valid_string_exposed_types)}"              )


from enum import Enum      class Operator(Enum):      """Enumeration of operators for Data Node filtering.        The possible values are:        - `EQUAL`      - `NOT_EQUAL`      - `LESS_THAN`      - `LESS_OR_EQUAL`      - `GREATER_THAN`      - `GREATER_OR_EQUAL`      """        EQUAL = 1      NOT_EQUAL = 2      LESS_THAN = 3      LESS_OR_EQUAL = 4      GREATER_THAN = 5      GREATER_OR_EQUAL = 6      class JoinOperator(Enum):      """      Enumeration of join operators for Data Node filtering. The possible values are `AND` and `OR`.      """        AND = 1      OR = 2


from .csv import CSVDataNode  from .data_node import DataNode  from .excel import ExcelDataNode  from .generic import GenericDataNode  from .in_memory import InMemoryDataNode  from .json import JSONDataNode  from .mongo import MongoCollectionDataNode  from .operator import JoinOperator, Operator  from .parquet import ParquetDataNode  from .pickle import PickleDataNode  from .sql import SQLDataNode  from .sql_table import SQLTableDataNode


import pathlib      class _AbstractFileDataNode(object):      """Abstract base class for data node implementations (CSVDataNode, ParquetDataNode, ExcelDataNode,      PickleDataNode and JSONDataNode) that are file based."""        __EXTENSION_MAP = {"csv": "csv", "excel": "xlsx", "parquet": "parquet", "pickle": "p", "json": "json"}        def _build_path(self, storage_type):          from taipy.config.config import Config            folder = f"{storage_type}s"          dir_path = pathlib.Path(Config.core.storage_folder) / folder          if not dir_path.exists():              dir_path.mkdir(parents=True, exist_ok=True)          return dir_path / f"{self.id}.{self.__EXTENSION_MAP.get(storage_type)}"


import os  from collections import defaultdict  from datetime import datetime, timedelta  from os.path import isfile  from typing import Any, Dict, List, Optional, Set, Tuple, Union    import modin.pandas as modin_pd  import numpy as np  import pandas as pd  from openpyxl import load_workbook    from taipy.config.common.scope import Scope    from .._backup._backup import _replace_in_backup_file  from .._entity._reload import _self_reload  from .._version._version_manager_factory import _VersionManagerFactory  from ..exceptions.exceptions import ExposedTypeLengthMismatch, NonExistingExcelSheet, SheetNameLengthMismatch  from ..job.job_id import JobId  from ._abstract_file import _AbstractFileDataNode  from ._abstract_tabular import _AbstractTabularDataNode  from .data_node import DataNode  from .data_node_id import DataNodeId, Edit      class ExcelDataNode(DataNode, _AbstractFileDataNode, _AbstractTabularDataNode):      """Data Node stored as an Excel file.        The Excel file format is _xlsx_.        Attributes:          config_id (str): Identifier of this data node configuration. It must be a valid Python              identifier.          scope (Scope^): The scope of this data node.          id (str): The unique identifier of this data node.          owner_id (str): The identifier of the owner (sequence_id, scenario_id, cycle_id) or              `None`.          parent_ids (Optional[Set[str]]): The identifiers of the parent tasks or `None`.          last_edit_date (datetime): The date and time of the last modification.          edits (List[Edit^]): The ordered list of edits for that job.          version (str): The string indicates the application version of the data node to instantiate. If not provided,              the current version is used.          validity_period (Optional[timedelta]): The duration implemented as a timedelta since the last edit date for              which the data node can be considered up-to-date. Once the validity period has passed, the data node is              considered stale and relevant tasks will run even if they are skippable (see the              [Task management page](../core/entities/task-mgt.md) for more details).              If _validity_period_ is set to `None`, the data node is always up-to-date.          edit_in_progress (bool): True if a task computing the data node has been submitted              and not completed yet. False otherwise.          editor_id (Optional[str]): The identifier of the user who is currently editing the data node.          editor_expiration_date (Optional[datetime]): The expiration date of the editor lock.          path (str): The path to the Excel file.          properties (dict[str, Any]): A dictionary of additional properties. The _properties_              must have a _"default_path"_ or _"path"_ entry with the path of the Excel file:                - _"default_path"_ `(str)`: The path of the Excel file.\n              - _"has_header"_ `(bool)`: If True, indicates that the Excel file has a header.\n              - _"sheet_name"_ `(Union[List[str], str])`: The list of sheet names to be used. This                  can be a unique name.\n              - _"exposed_type"_: The exposed type of the data read from Excel file. The default value is `pandas`.\n      """        __STORAGE_TYPE = "excel"      __EXPOSED_TYPE_PROPERTY = "exposed_type"      __EXPOSED_TYPE_NUMPY = "numpy"      __EXPOSED_TYPE_PANDAS = "pandas"      __EXPOSED_TYPE_MODIN = "modin"      __VALID_STRING_EXPOSED_TYPES = [__EXPOSED_TYPE_PANDAS, __EXPOSED_TYPE_MODIN, __EXPOSED_TYPE_NUMPY]      __PATH_KEY = "path"      __DEFAULT_DATA_KEY = "default_data"      __DEFAULT_PATH_KEY = "default_path"      __HAS_HEADER_PROPERTY = "has_header"      __SHEET_NAME_PROPERTY = "sheet_name"      _REQUIRED_PROPERTIES: List[str] = []        def __init__(          self,          config_id: str,          scope: Scope,          id: Optional[DataNodeId] = None,          owner_id: Optional[str] = None,          parent_ids: Optional[Set[str]] = None,          last_edit_date: Optional[datetime] = None,          edits: List[Edit] = None,          version: str = None,          validity_period: Optional[timedelta] = None,          edit_in_progress: bool = False,          editor_id: Optional[str] = None,          editor_expiration_date: Optional[datetime] = None,          properties: Dict = None,      ):          if properties is None:              properties = {}            default_value = properties.pop(self.__DEFAULT_DATA_KEY, None)          self._path = properties.get(self.__PATH_KEY, properties.get(self.__DEFAULT_PATH_KEY))          properties[self.__PATH_KEY] = self._path            if self.__SHEET_NAME_PROPERTY not in properties.keys():              properties[self.__SHEET_NAME_PROPERTY] = None          if self.__HAS_HEADER_PROPERTY not in properties.keys():              properties[self.__HAS_HEADER_PROPERTY] = True          if self.__EXPOSED_TYPE_PROPERTY not in properties.keys():              properties[self.__EXPOSED_TYPE_PROPERTY] = self.__EXPOSED_TYPE_PANDAS          self._check_exposed_type(properties[self.__EXPOSED_TYPE_PROPERTY], self.__VALID_STRING_EXPOSED_TYPES)            super().__init__(              config_id,              scope,              id,              owner_id,              parent_ids,              last_edit_date,              edits,              version or _VersionManagerFactory._build_manager()._get_latest_version(),              validity_period,              edit_in_progress,              editor_id,              editor_expiration_date,              **properties,          )          if not self._path:              self._path = self._build_path(self.storage_type())              properties[self.__PATH_KEY] = self._path            if default_value is not None and not os.path.exists(self._path):              self._write(default_value)              self._last_edit_date = datetime.now()              self._edits.append(                  Edit(                      {                          "timestamp": self._last_edit_date,                          "writer_identifier": "TAIPY",                          "comments": "Default data written.",                      }                  )              )            if not self._last_edit_date and isfile(self._path):              self._last_edit_date = datetime.now()            self._TAIPY_PROPERTIES.update(              {                  self.__EXPOSED_TYPE_PROPERTY,                  self.__PATH_KEY,                  self.__DEFAULT_PATH_KEY,                  self.__DEFAULT_DATA_KEY,                  self.__HAS_HEADER_PROPERTY,                  self.__SHEET_NAME_PROPERTY,              }          )        @property  # type: ignore      @_self_reload(DataNode._MANAGER_NAME)      def path(self):          return self._path        @path.setter      def path(self, value):          tmp_old_path = self._path          self._path = value          self.properties[self.__PATH_KEY] = value          _replace_in_backup_file(old_file_path=tmp_old_path, new_file_path=self._path)        @classmethod      def storage_type(cls) -> str:          return cls.__STORAGE_TYPE        @staticmethod      def _check_exposed_type(exposed_type, valid_string_exposed_types):          if isinstance(exposed_type, str):              _AbstractTabularDataNode._check_exposed_type(exposed_type, valid_string_exposed_types)          elif isinstance(exposed_type, list):              for t in exposed_type:                  _AbstractTabularDataNode._check_exposed_type(t, valid_string_exposed_types)          elif isinstance(exposed_type, dict):              for t in exposed_type.values():                  _AbstractTabularDataNode._check_exposed_type(t, valid_string_exposed_types)        def _read(self):          if self.properties[self.__EXPOSED_TYPE_PROPERTY] == self.__EXPOSED_TYPE_PANDAS:              return self._read_as_pandas_dataframe()          if self.properties[self.__EXPOSED_TYPE_PROPERTY] == self.__EXPOSED_TYPE_MODIN:              return self._read_as_modin_dataframe()          if self.properties[self.__EXPOSED_TYPE_PROPERTY] == self.__EXPOSED_TYPE_NUMPY:              return self._read_as_numpy()          return self._read_as()        def __sheet_name_to_list(self, properties):          if properties[self.__SHEET_NAME_PROPERTY]:              sheet_names = properties[self.__SHEET_NAME_PROPERTY]          else:              excel_file = load_workbook(properties[self.__PATH_KEY])              sheet_names = excel_file.sheetnames              excel_file.close()          return sheet_names if isinstance(sheet_names, (List, Set, Tuple)) else [sheet_names]        def _read_as(self):          excel_file = load_workbook(self._path)          exposed_type = self.properties[self.__EXPOSED_TYPE_PROPERTY]          work_books = defaultdict()          sheet_names = excel_file.sheetnames          provided_sheet_names = self.__sheet_name_to_list(self.properties)            for sheet_name in provided_sheet_names:              if sheet_name not in sheet_names:                  raise NonExistingExcelSheet(sheet_name, self._path)            if isinstance(exposed_type, List):              if len(provided_sheet_names) != len(self.properties[self.__EXPOSED_TYPE_PROPERTY]):                  raise ExposedTypeLengthMismatch(                      f"Expected {len(provided_sheet_names)} exposed types, got "                      f"{len(self.properties[self.__EXPOSED_TYPE_PROPERTY])}"                  )            for i, sheet_name in enumerate(provided_sheet_names):              work_sheet = excel_file[sheet_name]              sheet_exposed_type = exposed_type                if not isinstance(sheet_exposed_type, str):                  if isinstance(exposed_type, dict):                      sheet_exposed_type = exposed_type.get(sheet_name, self.__EXPOSED_TYPE_PANDAS)                  elif isinstance(exposed_type, List):                      sheet_exposed_type = exposed_type[i]                    if isinstance(sheet_exposed_type, str):                      if sheet_exposed_type == self.__EXPOSED_TYPE_NUMPY:                          work_books[sheet_name] = self._read_as_pandas_dataframe(sheet_name).to_numpy()                      elif sheet_exposed_type == self.__EXPOSED_TYPE_PANDAS:                          work_books[sheet_name] = self._read_as_pandas_dataframe(sheet_name)                      continue                res = list()              for row in work_sheet.rows:                  res.append([col.value for col in row])              if self.properties[self.__HAS_HEADER_PROPERTY] and res:                  header = res.pop(0)                  for i, row in enumerate(res):                      res[i] = sheet_exposed_type(**dict([[h, r] for h, r in zip(header, row)]))              else:                  for i, row in enumerate(res):                      res[i] = sheet_exposed_type(*row)              work_books[sheet_name] = res            excel_file.close()            if len(provided_sheet_names) == 1:              return work_books[provided_sheet_names[0]]          return work_books        def _read_as_numpy(self):          sheets = self._read_as_pandas_dataframe()          if isinstance(sheets, dict):              return {sheet_name: df.to_numpy() for sheet_name, df in sheets.items()}          return sheets.to_numpy()        def _do_read_excel(self, engine, sheet_names, kwargs) -> pd.DataFrame:          df = pd.read_excel(              self._path,              sheet_name=sheet_names,              **kwargs,          )          # We are using pandas to load modin dataframes because of a modin issue          # https://github.com/modin-project/modin/issues/4924          if engine == "modin":              if isinstance(df, dict):  # Check if it s a multiple sheet Excel file                  for key, value in df.items():                      df[key] = modin_pd.DataFrame(value)                  return df              return modin_pd.DataFrame(df)          return df        def __get_sheet_names_and_header(self, sheet_names):          kwargs: Dict[str, Any] = {}          if sheet_names is None:              sheet_names = self.properties[self.__SHEET_NAME_PROPERTY]          if not self.properties[self.__HAS_HEADER_PROPERTY]:              kwargs["header"] = None          return sheet_names, kwargs        def _read_as_pandas_dataframe(self, sheet_names=None) -> Union[Dict[Union[int, str], pd.DataFrame], pd.DataFrame]:          sheet_names, kwargs = self.__get_sheet_names_and_header(sheet_names)          try:              return self._do_read_excel("pandas", sheet_names, kwargs)          except pd.errors.EmptyDataError:              return pd.DataFrame()        def _read_as_modin_dataframe(          self, sheet_names=None      ) -> Union[Dict[Union[int, str], modin_pd.DataFrame], modin_pd.DataFrame]:          sheet_names, kwargs = self.__get_sheet_names_and_header(sheet_names)          try:              if kwargs.get("header", None):                  return modin_pd.read_excel(                      self._path,                      sheet_name=sheet_names,                      **kwargs,                  )              else:                  return self._do_read_excel("modin", sheet_names, kwargs)          except pd.errors.EmptyDataError:              return modin_pd.DataFrame()        def __append_excel_with_single_sheet(self, append_excel_fct, *args, **kwargs):          sheet_name = self.properties.get(self.__SHEET_NAME_PROPERTY)            with pd.ExcelWriter(self._path, mode="a", engine="openpyxl", if_sheet_exists="overlay") as writer:              if sheet_name:                  if not isinstance(sheet_name, str):                      sheet_name = sheet_name[0]                  append_excel_fct(                      writer, *args, **kwargs, sheet_name=sheet_name, startrow=writer.sheets[sheet_name].max_row                  )              else:                  sheet_name = list(writer.sheets.keys())[0]                  append_excel_fct(writer, *args, **kwargs, startrow=writer.sheets[sheet_name].max_row)        def __append_excel_with_multiple_sheets(self, data: Any, columns: List[str] = None):          with pd.ExcelWriter(self._path, mode="a", engine="openpyxl", if_sheet_exists="overlay") as writer:              # Each key stands for a sheet name              for sheet_name in data.keys():                  if isinstance(data[sheet_name], np.ndarray):                      df = pd.DataFrame(data[sheet_name])                  else:                      df = data[sheet_name]                    if columns:                      data[sheet_name].columns = columns                    df.to_excel(                      writer, sheet_name=sheet_name, index=False, header=False, startrow=writer.sheets[sheet_name].max_row                  )        def _append(self, data: Any):          if isinstance(data, Dict) and all(              [isinstance(x, (pd.DataFrame, modin_pd.DataFrame, np.ndarray)) for x in data.values()]          ):              self.__append_excel_with_multiple_sheets(data)          elif isinstance(data, (pd.DataFrame, modin_pd.DataFrame)):              self.__append_excel_with_single_sheet(data.to_excel, index=False, header=False)          else:              self.__append_excel_with_single_sheet(pd.DataFrame(data).to_excel, index=False, header=False)        def __write_excel_with_single_sheet(self, write_excel_fct, *args, **kwargs):          sheet_name = self.properties.get(self.__SHEET_NAME_PROPERTY)          if sheet_name:              if not isinstance(sheet_name, str):                  if len(sheet_name) > 1:                      raise SheetNameLengthMismatch                  else:                      sheet_name = sheet_name[0]              write_excel_fct(*args, **kwargs, sheet_name=sheet_name)          else:              write_excel_fct(*args, **kwargs)        def __write_excel_with_multiple_sheets(self, data: Any, columns: List[str] = None):          with pd.ExcelWriter(self._path) as writer:              # Each key stands for a sheet name              for key in data.keys():                  if isinstance(data[key], np.ndarray):                      df = pd.DataFrame(data[key])                  else:                      df = data[key]                    if columns:                      data[key].columns = columns                    df.to_excel(writer, key, index=False)        def _write(self, data: Any):          if isinstance(data, Dict) and all(              [isinstance(x, (pd.DataFrame, modin_pd.DataFrame, np.ndarray)) for x in data.values()]          ):              self.__write_excel_with_multiple_sheets(data)          elif isinstance(data, (pd.DataFrame, modin_pd.DataFrame)):              self.__write_excel_with_single_sheet(data.to_excel, self._path, index=False)          else:              self.__write_excel_with_single_sheet(pd.DataFrame(data).to_excel, self._path, index=False)        def write_with_column_names(self, data: Any, columns: List[str] = None, job_id: Optional[JobId] = None):          """Write a set of columns.            Parameters:              data (Any): The data to write.              columns (List[str]): The list of column names to write.              job_id (JobId^): An optional identifier of the writer.          """          if isinstance(data, Dict) and all(              [isinstance(x, (pd.DataFrame, modin_pd.DataFrame, np.ndarray)) for x in data.values()]          ):              self.__write_excel_with_multiple_sheets(data, columns=columns)          else:              df = pd.DataFrame(data)              if columns:                  df.columns = columns              self.__write_excel_with_single_sheet(df.to_excel, self.path, index=False)          self.track_edit(timestamp=datetime.now(), job_id=job_id)


import os  from typing import Dict, Iterable, List, Optional, Set, Union    from taipy.config._config import _Config  from taipy.config.common.scope import Scope  from taipy.config.config import Config    from .._backup._backup import _append_to_backup_file, _remove_from_backup_file  from .._manager._manager import _Manager  from .._version._version_mixin import _VersionMixin  from ..config.data_node_config import DataNodeConfig  from ..cycle.cycle_id import CycleId  from ..exceptions.exceptions import InvalidDataNodeType  from ..notification import Event, EventEntityType, EventOperation, Notifier, _make_event  from ..scenario.scenario_id import ScenarioId  from ..sequence.sequence_id import SequenceId  from ._abstract_file import _AbstractFileDataNode  from ._data_fs_repository import _DataFSRepository  from .data_node import DataNode  from .data_node_id import DataNodeId  from .pickle import PickleDataNode      class _DataManager(_Manager[DataNode], _VersionMixin):      __DATA_NODE_CLASS_MAP = DataNode._class_map()  # type: ignore      _ENTITY_NAME = DataNode.__name__      _EVENT_ENTITY_TYPE = EventEntityType.DATA_NODE      _repository: _DataFSRepository      __NAME_KEY = "name"        @classmethod      def _bulk_get_or_create(          cls,          data_node_configs: List[DataNodeConfig],          cycle_id: Optional[CycleId] = None,          scenario_id: Optional[ScenarioId] = None,      ) -> Dict[DataNodeConfig, DataNode]:          data_node_configs = [Config.data_nodes[dnc.id] for dnc in data_node_configs]          dn_configs_and_owner_id = []          for dn_config in data_node_configs:              scope = dn_config.scope              owner_id: Union[Optional[SequenceId], Optional[ScenarioId], Optional[CycleId]]              if scope == Scope.SCENARIO:                  owner_id = scenario_id              elif scope == Scope.CYCLE:                  owner_id = cycle_id              else:                  owner_id = None              dn_configs_and_owner_id.append((dn_config, owner_id))            data_nodes = cls._repository._get_by_configs_and_owner_ids(              dn_configs_and_owner_id, cls._build_filters_with_version(None)          )            return {              dn_config: data_nodes.get((dn_config, owner_id)) or cls._create_and_set(dn_config, owner_id, None)              for dn_config, owner_id in dn_configs_and_owner_id          }        @classmethod      def _create_and_set(          cls, data_node_config: DataNodeConfig, owner_id: Optional[str], parent_ids: Optional[Set[str]]      ) -> DataNode:          data_node = cls.__create(data_node_config, owner_id, parent_ids)          cls._set(data_node)          if isinstance(data_node, _AbstractFileDataNode):              _append_to_backup_file(new_file_path=data_node._path)          Notifier.publish(_make_event(data_node, EventOperation.CREATION))          return data_node        @classmethod      def __create(          cls, data_node_config: DataNodeConfig, owner_id: Optional[str], parent_ids: Optional[Set[str]]      ) -> DataNode:          try:              version = cls._get_latest_version()              props = data_node_config._properties.copy()                if data_node_config.storage_type:                  storage_type = data_node_config.storage_type              else:                  storage_type = Config.sections[DataNodeConfig.name][_Config.DEFAULT_KEY].storage_type                return cls.__DATA_NODE_CLASS_MAP[storage_type](                  config_id=data_node_config.id,                  scope=data_node_config.scope or DataNodeConfig._DEFAULT_SCOPE,                  validity_period=data_node_config.validity_period,                  owner_id=owner_id,                  parent_ids=parent_ids,                  version=version,                  properties=props,              )          except KeyError:              raise InvalidDataNodeType(data_node_config.storage_type)        @classmethod      def _get_all(cls, version_number: Optional[str] = None) -> List[DataNode]:          """          Returns all entities.          """          filters = cls._build_filters_with_version(version_number)          return cls._repository._load_all(filters)        @classmethod      def _clean_pickle_file(cls, data_node: DataNode):          if not isinstance(data_node, PickleDataNode):              return          if data_node.is_generated and os.path.exists(data_node.path):              os.remove(data_node.path)        @classmethod      def _clean_pickle_files(cls, data_nodes: Iterable[DataNode]):          for data_node in data_nodes:              cls._clean_pickle_file(data_node)        @classmethod      def _remove_dn_file_path_in_backup_file(cls, data_node: DataNode):          if isinstance(data_node, _AbstractFileDataNode):              _remove_from_backup_file(to_remove_file_path=data_node.path)        @classmethod      def _remove_dn_file_paths_in_backup_file(cls, data_nodes: Iterable[DataNode]):          for data_node in data_nodes:              cls._remove_dn_file_path_in_backup_file(data_node)        @classmethod      def _delete(cls, data_node_id: DataNodeId):          data_node = cls._get(data_node_id, None)          if data_node:              cls._clean_pickle_file(data_node)              cls._remove_dn_file_path_in_backup_file(data_node)          super()._delete(data_node_id)        @classmethod      def _delete_many(cls, data_node_ids: Iterable[DataNodeId]):          data_nodes = []          for data_node_id in data_node_ids:              if data_node := cls._get(data_node_id):                  data_nodes.append(data_node)          cls._clean_pickle_files(data_nodes)          cls._remove_dn_file_paths_in_backup_file(data_nodes)          super()._delete_many(data_node_ids)        @classmethod      def _delete_all(cls):          data_nodes = cls._get_all()          cls._clean_pickle_files(data_nodes)          cls._remove_dn_file_paths_in_backup_file(data_nodes)          super()._delete_all()        @classmethod      def _delete_by_version(cls, version_number: str):          data_nodes = cls._get_all(version_number)          cls._clean_pickle_files(data_nodes)          cls._remove_dn_file_paths_in_backup_file(data_nodes)          cls._repository._delete_by(attribute="version", value=version_number)          Notifier.publish(              Event(EventEntityType.DATA_NODE, EventOperation.DELETION, metadata={"delete_by_version": version_number})          )        @classmethod      def _get_by_config_id(cls, config_id: str, version_number: Optional[str] = None) -> List[DataNode]:          """          Get all datanodes by its config id.          """          filters = cls._build_filters_with_version(version_number)          if not filters:              filters = [{}]          for fil in filters:              fil.update({"config_id": config_id})          return cls._repository._load_all(filters)


from datetime import datetime, timedelta  from typing import Any, Dict, List, Optional, Set, Tuple, Union    import modin.pandas as modin_pd  import numpy as np  import pandas as pd  from sqlalchemy import MetaData, Table    from taipy.config.common.scope import Scope    from .._version._version_manager_factory import _VersionManagerFactory  from ..exceptions.exceptions import MissingRequiredProperty  from ._abstract_sql import _AbstractSQLDataNode  from .data_node_id import DataNodeId, Edit      class SQLTableDataNode(_AbstractSQLDataNode):      """Data Node stored in a SQL table.        Attributes:          config_id (str): Identifier of the data node configuration. It must be a valid Python              identifier.          scope (Scope^): The scope of this data node.          id (str): The unique identifier of this data node.          owner_id (str): The identifier of the owner (sequence_id, scenario_id, cycle_id) or              None.          parent_ids (Optional[Set[str]]): The identifiers of the parent tasks or `None`.          last_edit_date (datetime): The date and time of the last modification.          edits (List[Edit^]): The ordered list of edits for that job.          version (str): The string indicates the application version of the data node to instantiate. If not provided,              the current version is used.          validity_period (Optional[timedelta]): The duration implemented as a timedelta since the last edit date for              which the data node can be considered up-to-date. Once the validity period has passed, the data node is              considered stale and relevant tasks will run even if they are skippable (see the              [Task management page](../core/entities/task-mgt.md) for more details).              If _validity_period_ is set to `None`, the data node is always up-to-date.          edit_in_progress (bool): True if a task computing the data node has been submitted              and not completed yet. False otherwise.          editor_id (Optional[str]): The identifier of the user who is currently editing the data node.          editor_expiration_date (Optional[datetime]): The expiration date of the editor lock.          properties (dict[str, Any]): A dictionary of additional properties. Note that the              _properties_ parameter must at least contain an entry for _"db_name"_, _"db_engine"_, _"table_name"_:                - _"db_name"_ `(str)`: The database name, or the name of the SQLite database file.              - _"db_engine"_ `(str)`: The database engine. For now, the accepted values are _"sqlite"_, _"mssql"_,                  _"mysql"_, or _"postgresql"_.              - _"table_name"_ `(str)`: The name of the SQL table.              - _"db_username"_ `(str)`: The database username.              - _"db_password"_ `(str)`: The database password.              - _"db_host"_ `(str)`: The database host. The default value is _"localhost"_.              - _"db_port"_ `(int)`: The database port. The default value is 1433.              - _"db_driver"_ `(str)`: The database driver.              - _"sqlite_folder_path"_ (str): The path to the folder that contains SQLite file. The default value                  is the current working folder.              - _"sqlite_file_extension"_ (str): The filename extension of the SQLite file. The default value is ".db".              - _"db_extra_args"_ `(Dict[str, Any])`: A dictionary of additional arguments to be passed into database                  connection string.              - _"exposed_type"_: The exposed type of the data read from SQL query. The default value is `pandas`.      """        __STORAGE_TYPE = "sql_table"      __TABLE_KEY = "table_name"        def __init__(          self,          config_id: str,          scope: Scope,          id: Optional[DataNodeId] = None,          owner_id: Optional[str] = None,          parent_ids: Optional[Set[str]] = None,          last_edit_date: Optional[datetime] = None,          edits: Optional[List[Edit]] = None,          version: Optional[str] = None,          validity_period: Optional[timedelta] = None,          edit_in_progress: bool = False,          editor_id: Optional[str] = None,          editor_expiration_date: Optional[datetime] = None,          properties: Optional[Dict] = None,      ):          if properties is None:              properties = {}          if properties.get(self.__TABLE_KEY) is None:              raise MissingRequiredProperty(f"Property {self.__TABLE_KEY} is not informed and is required.")          super().__init__(              config_id,              scope,              id=id,              owner_id=owner_id,              parent_ids=parent_ids,              last_edit_date=last_edit_date,              edits=edits,              version=version or _VersionManagerFactory._build_manager()._get_latest_version(),              validity_period=validity_period,              edit_in_progress=edit_in_progress,              editor_id=editor_id,              editor_expiration_date=editor_expiration_date,              properties=properties,          )          self._TAIPY_PROPERTIES.update({self.__TABLE_KEY})        @classmethod      def storage_type(cls) -> str:          return cls.__STORAGE_TYPE        def _get_base_read_query(self) -> str:          return f"SELECT * FROM {self.properties[self.__TABLE_KEY]}"        def _do_append(self, data, engine, connection) -> None:          self.__insert_data(data, engine, connection)        def _do_write(self, data, engine, connection) -> None:          self.__insert_data(data, engine, connection, delete_table=True)        def __insert_data(self, data, engine, connection, delete_table: bool = False) -> None:          """          Insert data into a SQL table.            Parameters:              data (List[Dict]): a list of dictionaries, where each dictionary represents a row of the table.              table: a SQLAlchemy object that represents a table.              connection: a SQLAlchemy connection to write the data.              delete_table (bool): indicates if the table should be deleted before inserting the data.          """          table = self._create_table(engine)          if isinstance(data, (modin_pd.DataFrame, pd.DataFrame)):              self.__insert_dataframe(data, table, connection, delete_table)              return            if isinstance(data, np.ndarray):              data = data.tolist()          if not isinstance(data, list):              data = [data]            if len(data) == 0:              self.__delete_all_rows(table, connection, delete_table)              return            if isinstance(data[0], (tuple, list)):              self.__insert_tuples(data, table, connection, delete_table)          elif isinstance(data[0], dict):              self.__insert_dicts(data, table, connection, delete_table)          # If data is a primitive type, it will be inserted as a tuple of one element.          else:              self.__insert_tuples([(x,) for x in data], table, connection, delete_table)        def _create_table(self, engine) -> Table:          return Table(              self.properties[self.__TABLE_KEY],              MetaData(),              autoload_with=engine,          )        @classmethod      def __insert_dicts(cls, data: List[Dict], table: Any, connection: Any, delete_table: bool) -> None:          """          This method will insert the data contained in a list of dictionaries into a table. The query itself is handled          by SQLAlchemy, so it's only needed to pass the correct data type.          """          cls.__delete_all_rows(table, connection, delete_table)          connection.execute(table.insert(), data)        @classmethod      def __insert_dataframe(          cls, df: Union[modin_pd.DataFrame, pd.DataFrame], table: Any, connection: Any, delete_table: bool      ) -> None:          cls.__insert_dicts(df.to_dict(orient="records"), table, connection, delete_table)        @classmethod      def __insert_tuples(cls, data: List[Union[Tuple, List]], table: Any, connection: Any, delete_table: bool) -> None:          """          This method will look up the length of the first object of the list and build the insert through          creation of a string of '?' equivalent to the length of the element. The '?' character is used as          placeholder for a tuple of same size.          """          cls.__delete_all_rows(table, connection, delete_table)          markers = ",".join("?" * len(data[0]))          ins = "INSERT INTO {tablename} VALUES ({markers})"          ins = ins.format(tablename=table.name, markers=markers)          connection.execute(ins, data)        @classmethod      def __delete_all_rows(cls, table: Any, connection: Any, delete_table: bool) -> None:          if delete_table:              connection.execute(table.delete())


import csv  import os  from datetime import datetime, timedelta  from os.path import isfile  from typing import Any, Dict, List, Optional, Set    import modin.pandas as modin_pd  import numpy as np  import pandas as pd    from taipy.config.common.scope import Scope    from .._backup._backup import _replace_in_backup_file  from .._entity._reload import _self_reload  from .._version._version_manager_factory import _VersionManagerFactory  from ..job.job_id import JobId  from ._abstract_file import _AbstractFileDataNode  from ._abstract_tabular import _AbstractTabularDataNode  from .data_node import DataNode  from .data_node_id import DataNodeId, Edit      class CSVDataNode(DataNode, _AbstractFileDataNode, _AbstractTabularDataNode):      """Data Node stored as a CSV file.        Attributes:          config_id (str): Identifier of the data node configuration. This string must be a valid              Python identifier.          scope (Scope^): The scope of this data node.          id (str): The unique identifier of this data node.          owner_id (str): The identifier of the owner (sequence_id, scenario_id, cycle_id) or `None`.          parent_ids (Optional[Set[str]]): The identifiers of the parent tasks or `None`.          last_edit_date (datetime): The date and time of the last modification.          edits (List[Edit^]): The ordered list of edits for that job.          version (str): The string indicates the application version of the data node to instantiate. If not provided,              the current version is used.          validity_period (Optional[timedelta]): The duration implemented as a timedelta since the last edit date for              which the data node can be considered up-to-date. Once the validity period has passed, the data node is              considered stale and relevant tasks will run even if they are skippable (see the              [Task management page](../core/entities/task-mgt.md) for more details).              If _validity_period_ is set to `None`, the data node is always up-to-date.          edit_in_progress (bool): True if a task computing the data node has been submitted              and not completed yet. False otherwise.          editor_id (Optional[str]): The identifier of the user who is currently editing the data node.          editor_expiration_date (Optional[datetime]): The expiration date of the editor lock.          path (str): The path to the CSV file.          properties (dict[str, Any]): A dictionary of additional properties. The _properties_              must have a _"default_path"_ or _"path"_ entry with the path of the CSV file:                - _"default_path"_ `(str)`: The default path of the CSV file.\n              - _"encoding"_ `(str)`: The encoding of the CSV file. The default value is `utf-8`.\n              - _"default_data"_: The default data of the data nodes instantiated from this csv data node.\n              - _"has_header"_ `(bool)`: If True, indicates that the CSV file has a header.\n              - _"exposed_type"_: The exposed type of the data read from CSV file. The default value is `pandas`.\n      """        __STORAGE_TYPE = "csv"      __EXPOSED_TYPE_PROPERTY = "exposed_type"      __EXPOSED_TYPE_NUMPY = "numpy"      __EXPOSED_TYPE_PANDAS = "pandas"      __EXPOSED_TYPE_MODIN = "modin"      __VALID_STRING_EXPOSED_TYPES = [__EXPOSED_TYPE_PANDAS, __EXPOSED_TYPE_MODIN, __EXPOSED_TYPE_NUMPY]      __PATH_KEY = "path"      __DEFAULT_PATH_KEY = "default_path"      __ENCODING_KEY = "encoding"      __DEFAULT_DATA_KEY = "default_data"      __HAS_HEADER_PROPERTY = "has_header"      _REQUIRED_PROPERTIES: List[str] = []        def __init__(          self,          config_id: str,          scope: Scope,          id: Optional[DataNodeId] = None,          owner_id: Optional[str] = None,          parent_ids: Optional[Set[str]] = None,          last_edit_date: Optional[datetime] = None,          edits: Optional[List[Edit]] = None,          version: Optional[str] = None,          validity_period: Optional[timedelta] = None,          edit_in_progress: bool = False,          editor_id: Optional[str] = None,          editor_expiration_date: Optional[datetime] = None,          properties: Optional[Dict] = None,      ):          if properties is None:              properties = {}            default_value = properties.pop(self.__DEFAULT_DATA_KEY, None)            if self.__ENCODING_KEY not in properties.keys():              properties[self.__ENCODING_KEY] = "utf-8"            if self.__HAS_HEADER_PROPERTY not in properties.keys():              properties[self.__HAS_HEADER_PROPERTY] = True            if self.__EXPOSED_TYPE_PROPERTY not in properties.keys():              properties[self.__EXPOSED_TYPE_PROPERTY] = self.__EXPOSED_TYPE_PANDAS          self._check_exposed_type(properties[self.__EXPOSED_TYPE_PROPERTY], self.__VALID_STRING_EXPOSED_TYPES)            super().__init__(              config_id,              scope,              id,              owner_id,              parent_ids,              last_edit_date,              edits,              version or _VersionManagerFactory._build_manager()._get_latest_version(),              validity_period,              edit_in_progress,              editor_id,              editor_expiration_date,              **properties,          )          self._path = properties.get(self.__PATH_KEY, properties.get(self.__DEFAULT_PATH_KEY))          if not self._path:              self._path = self._build_path(self.storage_type())          properties[self.__PATH_KEY] = self._path            if not self._last_edit_date and isfile(self._path):              self._last_edit_date = datetime.now()          if default_value is not None and not os.path.exists(self._path):              self._write(default_value)              self._last_edit_date = datetime.now()              self._edits.append(                  Edit(                      {                          "timestamp": self._last_edit_date,                          "writer_identifier": "TAIPY",                          "comments": "Default data written.",                      }                  )              )            self._TAIPY_PROPERTIES.update(              {                  self.__EXPOSED_TYPE_PROPERTY,                  self.__PATH_KEY,                  self.__DEFAULT_PATH_KEY,                  self.__ENCODING_KEY,                  self.__DEFAULT_DATA_KEY,                  self.__HAS_HEADER_PROPERTY,              }          )        @classmethod      def storage_type(cls) -> str:          return cls.__STORAGE_TYPE        @property  # type: ignore      @_self_reload(DataNode._MANAGER_NAME)      def path(self):          return self._path        @path.setter      def path(self, value):          tmp_old_path = self._path          self._path = value          self.properties[self.__PATH_KEY] = value          _replace_in_backup_file(old_file_path=tmp_old_path, new_file_path=self._path)        def _read(self):          if self.properties[self.__EXPOSED_TYPE_PROPERTY] == self.__EXPOSED_TYPE_PANDAS:              return self._read_as_pandas_dataframe()          if self.properties[self.__EXPOSED_TYPE_PROPERTY] == self.__EXPOSED_TYPE_MODIN:              return self._read_as_modin_dataframe()          if self.properties[self.__EXPOSED_TYPE_PROPERTY] == self.__EXPOSED_TYPE_NUMPY:              return self._read_as_numpy()          return self._read_as()        def _read_as(self):          custom_class = self.properties[self.__EXPOSED_TYPE_PROPERTY]          with open(self._path, encoding=self.properties[self.__ENCODING_KEY]) as csvFile:              res = list()              if self.properties[self.__HAS_HEADER_PROPERTY]:                  reader = csv.DictReader(csvFile)                  for line in reader:                      res.append(custom_class(**line))              else:                  reader = csv.reader(                      csvFile,                  )                  for line in reader:                      res.append(custom_class(*line))              return res        def _read_as_numpy(self) -> np.ndarray:          return self._read_as_pandas_dataframe().to_numpy()        def _read_as_pandas_dataframe(          self, usecols: Optional[List[int]] = None, column_names: Optional[List[str]] = None      ) -> pd.DataFrame:          try:              if self.properties[self.__HAS_HEADER_PROPERTY]:                  if column_names:                      return pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY])[column_names]                  return pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY])              else:                  if usecols:                      return pd.read_csv(                          self._path, encoding=self.properties[self.__ENCODING_KEY], header=None, usecols=usecols                      )                  return pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY], header=None)          except pd.errors.EmptyDataError:              return pd.DataFrame()        def _read_as_modin_dataframe(          self, usecols: Optional[List[int]] = None, column_names: Optional[List[str]] = None      ) -> modin_pd.DataFrame:          try:              if self.properties[self.__HAS_HEADER_PROPERTY]:                  if column_names:                      return modin_pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY])[column_names]                  return modin_pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY])              else:                  if usecols:                      return modin_pd.read_csv(                          self._path, header=None, usecols=usecols, encoding=self.properties[self.__ENCODING_KEY]                      )                  return modin_pd.read_csv(self._path, header=None, encoding=self.properties[self.__ENCODING_KEY])          except pd.errors.EmptyDataError:              return modin_pd.DataFrame()        def _append(self, data: Any):          if isinstance(data, (pd.DataFrame, modin_pd.DataFrame)):              data.to_csv(self._path, mode="a", index=False, encoding=self.properties[self.__ENCODING_KEY], header=False)          else:              pd.DataFrame(data).to_csv(                  self._path, mode="a", index=False, encoding=self.properties[self.__ENCODING_KEY], header=False              )        def _write(self, data: Any):          if isinstance(data, (pd.DataFrame, modin_pd.DataFrame)):              data.to_csv(self._path, index=False, encoding=self.properties[self.__ENCODING_KEY])          else:              pd.DataFrame(data).to_csv(self._path, index=False, encoding=self.properties[self.__ENCODING_KEY])        def write_with_column_names(self, data: Any, columns: Optional[List[str]] = None, job_id: Optional[JobId] = None):          """Write a selection of columns.            Parameters:              data (Any): The data to write.              columns (Optional[List[str]]): The list of column names to write.              job_id (JobId^): An optional identifier of the writer.          """          if not columns:              df = pd.DataFrame(data)          else:              df = pd.DataFrame(data, columns=columns)          df.to_csv(self._path, index=False, encoding=self.properties[self.__ENCODING_KEY])          self.track_edit(timestamp=datetime.now(), job_id=job_id)


from typing import Type    from .._manager._manager_factory import _ManagerFactory  from ..common._utils import _load_fct  from ._data_fs_repository import _DataFSRepository  from ._data_manager import _DataManager  from ._data_sql_repository import _DataSQLRepository      class _DataManagerFactory(_ManagerFactory):        __REPOSITORY_MAP = {"default": _DataFSRepository, "sql": _DataSQLRepository}        @classmethod      def _build_manager(cls) -> Type[_DataManager]:  # type: ignore          if cls._using_enterprise():              data_manager = _load_fct(                  cls._TAIPY_ENTERPRISE_CORE_MODULE + ".data._data_manager", "_DataManager"              )  # type: ignore              build_repository = _load_fct(                  cls._TAIPY_ENTERPRISE_CORE_MODULE + ".data._data_manager_factory", "_DataManagerFactory"              )._build_repository  # type: ignore          else:              data_manager = _DataManager              build_repository = cls._build_repository          data_manager._repository = build_repository()  # type: ignore          return data_manager  # type: ignore        @classmethod      def _build_repository(cls):          return cls._get_repository_with_repo_map(cls.__REPOSITORY_MAP)()


import os  import pickle  from datetime import datetime, timedelta  from typing import Any, List, Optional, Set    import modin.pandas as pd    from taipy.config.common.scope import Scope    from .._backup._backup import _replace_in_backup_file  from .._entity._reload import _self_reload  from .._version._version_manager_factory import _VersionManagerFactory  from ._abstract_file import _AbstractFileDataNode  from .data_node import DataNode  from .data_node_id import DataNodeId, Edit      class PickleDataNode(DataNode, _AbstractFileDataNode):      """Data Node stored as a pickle file.        Attributes:          config_id (str): Identifier of the data node configuration. It must be a valid Python              identifer.          scope (Scope^): The scope of this data node.          id (str): The unique identifier of this data node.          owner_id (str): The identifier of the owner (sequence_id, scenario_id, cycle_id) or              `None`.          parent_ids (Optional[Set[str]]): The identifiers of the parent tasks or `None`.          last_edit_date (datetime): The date and time of the last modification.          edits (List[Edit^]): The ordered list of edits for that job.          version (str): The string indicates the application version of the data node to instantiate. If not provided,              the current version is used.          validity_period (Optional[timedelta]): The duration implemented as a timedelta since the last edit date for              which the data node can be considered up-to-date. Once the validity period has passed, the data node is              considered stale and relevant tasks will run even if they are skippable (see the              [Task management page](../core/entities/task-mgt.md) for more details).              If _validity_period_ is set to `None`, the data node is always up-to-date.          edit_in_progress (bool): True if a task computing the data node has been submitted              and not completed yet. False otherwise.          editor_id (Optional[str]): The identifier of the user who is currently editing the data node.          editor_expiration_date (Optional[datetime]): The expiration date of the editor lock.          properties (dict[str, Any]): A dictionary of additional properties.              When creating a pickle data node, if the _properties_ dictionary contains a              _"default_data"_ entry, the data node is automatically written with the corresponding              _"default_data"_ value.              If the _properties_ dictionary contains a _"default_path"_ or _"path"_ entry, the data will be stored              using the corresponding value as the name of the pickle file.      """        __STORAGE_TYPE = "pickle"      __PATH_KEY = "path"      __DEFAULT_PATH_KEY = "default_path"      __DEFAULT_DATA_KEY = "default_data"      __IS_GENERATED_KEY = "is_generated"      _REQUIRED_PROPERTIES: List[str] = []        def __init__(          self,          config_id: str,          scope: Scope,          id: Optional[DataNodeId] = None,          owner_id: Optional[str] = None,          parent_ids: Optional[Set[str]] = None,          last_edit_date: Optional[datetime] = None,          edits: Optional[List[Edit]] = None,          version: str = None,          validity_period: Optional[timedelta] = None,          edit_in_progress: bool = False,          editor_id: Optional[str] = None,          editor_expiration_date: Optional[datetime] = None,          properties=None,      ):          if properties is None:              properties = {}          default_value = properties.pop(self.__DEFAULT_DATA_KEY, None)          self._path = properties.get(self.__PATH_KEY, properties.get(self.__DEFAULT_PATH_KEY))          if self._path is not None:              properties[self.__PATH_KEY] = self._path          self._is_generated = properties.get(self.__IS_GENERATED_KEY, self._path is None)          properties[self.__IS_GENERATED_KEY] = self._is_generated          super().__init__(              config_id,              scope,              id,              owner_id,              parent_ids,              last_edit_date,              edits,              version or _VersionManagerFactory._build_manager()._get_latest_version(),              validity_period,              edit_in_progress,              editor_id,              editor_expiration_date,              **properties,          )          if self._path is None:              self._path = self._build_path(self.storage_type())          if not self._last_edit_date and os.path.exists(self._path):              self._last_edit_date = datetime.now()          if default_value is not None and not os.path.exists(self._path):              self._write(default_value)              self._last_edit_date = datetime.now()              self._edits.append(                  Edit(                      {                          "timestamp": self._last_edit_date,                          "writer_identifier": "TAIPY",                          "comments": "Default data written.",                      }                  )              )            self._TAIPY_PROPERTIES.update(              {                  self.__PATH_KEY,                  self.__DEFAULT_PATH_KEY,                  self.__DEFAULT_DATA_KEY,                  self.__IS_GENERATED_KEY,              }          )        @classmethod      def storage_type(cls) -> str:          return cls.__STORAGE_TYPE        @property  # type: ignore      @_self_reload(DataNode._MANAGER_NAME)      def path(self) -> Any:          return self._path        @path.setter      def path(self, value):          tmp_old_path = self._path          self._path = value          self.properties[self.__PATH_KEY] = value          self.properties[self.__IS_GENERATED_KEY] = False          _replace_in_backup_file(old_file_path=tmp_old_path, new_file_path=self._path)        @property  # type: ignore      @_self_reload(DataNode._MANAGER_NAME)      def is_generated(self) -> bool:          return self._is_generated        def _read(self):          os.environ["MODIN_PERSISTENT_PICKLE"] = "True"          with open(self._path, "rb") as pf:              return pickle.load(pf)        def _write(self, data):          if isinstance(data, (pd.DataFrame, pd.Series)):              os.environ["MODIN_PERSISTENT_PICKLE"] = "True"          with open(self._path, "wb") as pf:              pickle.dump(data, pf)


from datetime import datetime, timedelta  from typing import Dict, List, Optional, Set    from sqlalchemy import text    from taipy.config.common.scope import Scope    from .._version._version_manager_factory import _VersionManagerFactory  from ..exceptions.exceptions import MissingAppendQueryBuilder, MissingRequiredProperty  from ._abstract_sql import _AbstractSQLDataNode  from .data_node_id import DataNodeId, Edit      class SQLDataNode(_AbstractSQLDataNode):      """Data Node stored in a SQL database.        Attributes:          config_id (str): Identifier of the data node configuration. It must be a valid Python              identifier.          scope (Scope^): The scope of this data node.          id (str): The unique identifier of this data node.          owner_id (str): The identifier of the owner (sequence_id, scenario_id, cycle_id) or              None.          parent_ids (Optional[Set[str]]): The identifiers of the parent tasks or `None`.          last_edit_date (datetime): The date and time of the last modification.          edits (List[Edit^]): The ordered list of edits for that job.          version (str): The string indicates the application version of the data node to instantiate. If not provided,              the current version is used.          validity_period (Optional[timedelta]): The duration implemented as a timedelta since the last edit date for              which the data node can be considered up-to-date. Once the validity period has passed, the data node is              considered stale and relevant tasks will run even if they are skippable (see the              [Task management page](../core/entities/task-mgt.md) for more details).              If _validity_period_ is set to `None`, the data node is always up-to-date.          edit_in_progress (bool): True if a task computing the data node has been submitted              and not completed yet. False otherwise.          editor_id (Optional[str]): The identifier of the user who is currently editing the data node.          editor_expiration_date (Optional[datetime]): The expiration date of the editor lock.          properties (dict[str, Any]): A dictionary of additional properties. Note that the              _properties_ parameter must at least contain an entry for _"db_name"_, _"db_engine"_, _"read_query"_,              and _"write_query_builder"_:                - _"db_name"_ `(str)`: The database name, or the name of the SQLite database file.              - _"db_engine"_ `(str)`: The database engine. Possible values are _"sqlite"_, _"mssql"_, _"mysql"_, or                  _"postgresql"_.              - _"read_query"_ `(str)`: The SQL query string used to read the data from the database.              - _"write_query_builder"_ `(Callable)`: A callback function that takes the data as an input parameter and                  returns a list of SQL queries to be executed when writing data to the data node.              - _"append_query_builder"_ `(Callable)`: A callback function that takes the data as an input parameter and                  returns a list of SQL queries to be executed when appending data to the data node.              - _"db_username"_ `(str)`: The database username.              - _"db_password"_ `(str)`: The database password.              - _"db_host"_ `(str)`: The database host. The default value is _"localhost"_.              - _"db_port"_ `(int)`: The database port. The default value is 1433.              - _"db_driver"_ `(str)`: The database driver.              - _"sqlite_folder_path"_ (str): The path to the folder that contains SQLite file. The default value                  is the current working folder.              - _"sqlite_file_extension"_ (str): The filename extension of the SQLite file. The default value is ".db".              - _"db_extra_args"_ `(Dict[str, Any])`: A dictionary of additional arguments to be passed into database                  connection string.              - _"exposed_type"_: The exposed type of the data read from SQL query. The default value is `pandas`.      """        __STORAGE_TYPE = "sql"      __READ_QUERY_KEY = "read_query"      _WRITE_QUERY_BUILDER_KEY = "write_query_builder"      _APPEND_QUERY_BUILDER_KEY = "append_query_builder"        def __init__(          self,          config_id: str,          scope: Scope,          id: Optional[DataNodeId] = None,          owner_id: Optional[str] = None,          parent_ids: Optional[Set[str]] = None,          last_edit_date: Optional[datetime] = None,          edits: Optional[List[Edit]] = None,          version: Optional[str] = None,          validity_period: Optional[timedelta] = None,          edit_in_progress: bool = False,          editor_id: Optional[str] = None,          editor_expiration_date: Optional[datetime] = None,          properties: Optional[Dict] = None,      ):          if properties is None:              properties = {}          if properties.get(self.__READ_QUERY_KEY) is None:              raise MissingRequiredProperty(f"Property {self.__READ_QUERY_KEY} is not informed and is required.")          if properties.get(self._WRITE_QUERY_BUILDER_KEY) is None:              raise MissingRequiredProperty(f"Property {self._WRITE_QUERY_BUILDER_KEY} is not informed and is required.")            super().__init__(              config_id,              scope,              id,              owner_id,              parent_ids,              last_edit_date,              edits,              version or _VersionManagerFactory._build_manager()._get_latest_version(),              validity_period,              edit_in_progress,              editor_id,              editor_expiration_date,              properties=properties,          )            self._TAIPY_PROPERTIES.update(              {                  self.__READ_QUERY_KEY,                  self._WRITE_QUERY_BUILDER_KEY,                  self._APPEND_QUERY_BUILDER_KEY,              }          )        @classmethod      def storage_type(cls) -> str:          return cls.__STORAGE_TYPE        def _get_base_read_query(self) -> str:          return self.properties.get(self.__READ_QUERY_KEY)        def _do_append(self, data, engine, connection) -> None:          if not self.properties.get(self._APPEND_QUERY_BUILDER_KEY):              raise MissingAppendQueryBuilder            queries = self.properties.get(self._APPEND_QUERY_BUILDER_KEY)(data)          self.__execute_queries(queries, connection)        def _do_write(self, data, engine, connection) -> None:          queries = self.properties.get(self._WRITE_QUERY_BUILDER_KEY)(data)          self.__execute_queries(queries, connection)        def __execute_queries(self, queries, connection) -> None:          if not isinstance(queries, List):              queries = [queries]          for query in queries:              if isinstance(query, str):                  connection.execute(text(query))              else:                  statement = query[0]                  parameters = query[1]                  connection.execute(text(statement), parameters)


#  # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with  # the License. You may obtain a copy of the License at  #  #        http://www.apache.org/licenses/LICENSE-2.0  #  # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on  # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the  # specific language governing permissions and limitations under the License.  from .._repository._sql_repository import _SQLRepository  from ._data_converter import _DataNodeConverter  from ._data_model import _DataNodeModel      class _DataSQLRepository(_SQLRepository):      def __init__(self):          super().__init__(model_type=_DataNodeModel, converter=_DataNodeConverter)