Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /MTSWebServices_syncmaster /syncmaster /worker /controller.py
| # SPDX-FileCopyrightText: 2023-present MTS PJSC | |
| # SPDX-License-Identifier: Apache-2.0 | |
| import logging | |
| from collections.abc import Callable | |
| from tempfile import TemporaryDirectory | |
| from typing import Any | |
| from etl_entities.hwm_store import BaseHWMStore | |
| from horizon.client.auth import LoginPassword | |
| from horizon_hwm_store import HorizonHWMStore | |
| from onetl.hooks import slot, support_hooks | |
| from onetl.strategy import IncrementalStrategy | |
| from syncmaster.db.models import Connection, Run | |
| from syncmaster.dto.connections import ( | |
| ClickhouseConnectionDTO, | |
| ConnectionDTO, | |
| FTPConnectionDTO, | |
| FTPSConnectionDTO, | |
| HDFSConnectionDTO, | |
| HiveConnectionDTO, | |
| MSSQLConnectionDTO, | |
| MySQLConnectionDTO, | |
| OracleConnectionDTO, | |
| PostgresConnectionDTO, | |
| S3ConnectionDTO, | |
| SambaConnectionDTO, | |
| SFTPConnectionDTO, | |
| WebDAVConnectionDTO, | |
| get_iceberg_connection_dto, | |
| ) | |
| from syncmaster.dto.runs import RunDTO | |
| from syncmaster.dto.transfers import ( | |
| ClickhouseTransferDTO, | |
| DBTransferDTO, | |
| FileTransferDTO, | |
| FTPSTransferDTO, | |
| FTPTransferDTO, | |
| HDFSTransferDTO, | |
| HiveTransferDTO, | |
| IcebergTransferDTO, | |
| MSSQLTransferDTO, | |
| MySQLTransferDTO, | |
| OracleTransferDTO, | |
| PostgresTransferDTO, | |
| S3TransferDTO, | |
| SambaTransferDTO, | |
| SFTPTransferDTO, | |
| WebDAVTransferDTO, | |
| ) | |
| from syncmaster.dto.transfers_resources import Resources | |
| from syncmaster.dto.transfers_strategy import Strategy | |
| from syncmaster.exceptions.connection import ConnectionTypeNotRecognizedError | |
| from syncmaster.worker.handlers.base import Handler | |
| from syncmaster.worker.handlers.db.clickhouse import ClickhouseHandler | |
| from syncmaster.worker.handlers.db.hive import HiveHandler | |
| from syncmaster.worker.handlers.db.iceberg import IcebergRESTCatalogS3Handler | |
| from syncmaster.worker.handlers.db.mssql import MSSQLHandler | |
| from syncmaster.worker.handlers.db.mysql import MySQLHandler | |
| from syncmaster.worker.handlers.db.oracle import OracleHandler | |
| from syncmaster.worker.handlers.db.postgres import PostgresHandler | |
| from syncmaster.worker.handlers.file.ftp import FTPHandler | |
| from syncmaster.worker.handlers.file.ftps import FTPSHandler | |
| from syncmaster.worker.handlers.file.hdfs import HDFSHandler | |
| from syncmaster.worker.handlers.file.s3 import S3Handler | |
| from syncmaster.worker.handlers.file.samba import SambaHandler | |
| from syncmaster.worker.handlers.file.sftp import SFTPHandler | |
| from syncmaster.worker.handlers.file.webdav import WebDAVHandler | |
| from syncmaster.worker.settings import WorkerAppSettings | |
| logger = logging.getLogger(__name__) | |
| HANDLERS_MAPPING = dict[ | |
| str, | |
| tuple[ | |
| type[Handler], | |
| Callable[..., ConnectionDTO], | |
| type[DBTransferDTO] | type[FileTransferDTO], | |
| type[RunDTO], | |
| ], | |
| ] | |
| connection_handler_proxy: HANDLERS_MAPPING = { | |
| "hive": ( | |
| HiveHandler, | |
| HiveConnectionDTO, | |
| HiveTransferDTO, | |
| RunDTO, | |
| ), | |
| "iceberg": ( | |
| IcebergRESTCatalogS3Handler, | |
| get_iceberg_connection_dto, | |
| IcebergTransferDTO, | |
| RunDTO, | |
| ), | |
| "oracle": ( | |
| OracleHandler, | |
| OracleConnectionDTO, | |
| OracleTransferDTO, | |
| RunDTO, | |
| ), | |
| "clickhouse": ( | |
| ClickhouseHandler, | |
| ClickhouseConnectionDTO, | |
| ClickhouseTransferDTO, | |
| RunDTO, | |
| ), | |
| "mssql": ( | |
| MSSQLHandler, | |
| MSSQLConnectionDTO, | |
| MSSQLTransferDTO, | |
| RunDTO, | |
| ), | |
| "mysql": ( | |
| MySQLHandler, | |
| MySQLConnectionDTO, | |
| MySQLTransferDTO, | |
| RunDTO, | |
| ), | |
| "postgres": ( | |
| PostgresHandler, | |
| PostgresConnectionDTO, | |
| PostgresTransferDTO, | |
| RunDTO, | |
| ), | |
| "s3": ( | |
| S3Handler, | |
| S3ConnectionDTO, | |
| S3TransferDTO, | |
| RunDTO, | |
| ), | |
| "hdfs": ( | |
| HDFSHandler, | |
| HDFSConnectionDTO, | |
| HDFSTransferDTO, | |
| RunDTO, | |
| ), | |
| "sftp": ( | |
| SFTPHandler, | |
| SFTPConnectionDTO, | |
| SFTPTransferDTO, | |
| RunDTO, | |
| ), | |
| "ftp": ( | |
| FTPHandler, | |
| FTPConnectionDTO, | |
| FTPTransferDTO, | |
| RunDTO, | |
| ), | |
| "ftps": ( | |
| FTPSHandler, | |
| FTPSConnectionDTO, | |
| FTPSTransferDTO, | |
| RunDTO, | |
| ), | |
| "samba": ( | |
| SambaHandler, | |
| SambaConnectionDTO, | |
| SambaTransferDTO, | |
| RunDTO, | |
| ), | |
| "webdav": ( | |
| WebDAVHandler, | |
| WebDAVConnectionDTO, | |
| WebDAVTransferDTO, | |
| RunDTO, | |
| ), | |
| } | |
| class TransferController: | |
| settings: WorkerAppSettings | |
| source_handler: Handler | |
| target_handler: Handler | |
| def __init__( # noqa: PLR0913, PLR0917 | |
| self, | |
| settings: WorkerAppSettings, | |
| run: Run, | |
| source_connection: Connection, | |
| source_auth_data: dict, | |
| target_connection: Connection, | |
| target_auth_data: dict, | |
| ): | |
| self.temp_dir = TemporaryDirectory(prefix=f"syncmaster_{run.id}_") | |
| self.settings = settings | |
| self.run = run | |
| self.source_handler = self.get_handler( | |
| connection_data=source_connection.data, | |
| run_data={"id": run.id, "created_at": run.created_at}, | |
| transfer_id=run.transfer.id, | |
| transfer_name=run.transfer.name, | |
| group_name=run.transfer.group.name, | |
| transfer_params=run.transfer.source_params, | |
| strategy_params=run.transfer.strategy_params, | |
| resources=run.transfer.resources, | |
| transformations=run.transfer.transformations, | |
| connection_auth_data=source_auth_data, | |
| temp_dir=TemporaryDirectory(dir=self.temp_dir.name, prefix="downloaded_"), | |
| ) | |
| self.target_handler = self.get_handler( | |
| connection_data=target_connection.data, | |
| run_data={"id": run.id, "created_at": run.created_at}, | |
| transfer_id=run.transfer.id, | |
| transfer_name=run.transfer.name, | |
| group_name=run.transfer.group.name, | |
| transfer_params=run.transfer.target_params, | |
| strategy_params=run.transfer.strategy_params, | |
| resources=run.transfer.resources, | |
| transformations=run.transfer.transformations, | |
| connection_auth_data=target_auth_data, | |
| temp_dir=TemporaryDirectory(dir=self.temp_dir.name, prefix="written_"), | |
| ) | |
| def perform_transfer(self) -> None: | |
| try: | |
| spark = self.settings.worker.create_spark_session_function( | |
| run=self.run, | |
| source=self.source_handler.connection_dto, | |
| target=self.target_handler.connection_dto, | |
| settings=self.settings.worker, | |
| ) | |
| with spark: | |
| self.source_handler.connect(spark) | |
| self.target_handler.connect(spark) | |
| if self.source_handler.transfer_dto.strategy.type == "incremental": | |
| return self._perform_incremental_transfer() | |
| if self.source_handler.transfer_dto.strategy.type == "full" and self.settings.hwm_store.enabled: | |
| self._reset_transfer_hwm() | |
| df = self.source_handler.read() | |
| self.target_handler.write(df) | |
| finally: | |
| self.temp_dir.cleanup() | |
| def get_handler( # noqa: PLR0913, PLR0917 | |
| self, | |
| connection_data: dict[str, Any], | |
| connection_auth_data: dict, | |
| run_data: dict[str, Any], | |
| transfer_id: int, | |
| transfer_name: str, | |
| group_name: str, | |
| transfer_params: dict[str, Any], | |
| strategy_params: dict[str, Any], | |
| resources: dict[str, Any], | |
| transformations: list[dict], | |
| temp_dir: TemporaryDirectory, | |
| ) -> Handler: | |
| connection_data.update(connection_auth_data) | |
| connection_data.pop("type") | |
| handler_type = transfer_params.pop("type", None) | |
| if connection_handler_proxy.get(handler_type, None) is None: | |
| raise ConnectionTypeNotRecognizedError | |
| handler, connection_dto, transfer_dto, run_dto = connection_handler_proxy[handler_type] | |
| return handler( | |
| connection_dto=connection_dto(**connection_data), | |
| transfer_dto=transfer_dto( | |
| id=transfer_id, | |
| name=transfer_name, | |
| group_name=group_name, | |
| strategy=Strategy.from_dict(strategy_params), | |
| resources=Resources(**resources), | |
| transformations=transformations, | |
| **transfer_params, | |
| ), | |
| run_dto=run_dto(**run_data), | |
| temp_dir=temp_dir, | |
| ) | |
| def _get_hwm_store(self) -> BaseHWMStore: | |
| return HorizonHWMStore( | |
| api_url=self.settings.hwm_store.url, # type: ignore[arg-type] | |
| auth=LoginPassword( | |
| login=self.settings.hwm_store.user, # type: ignore[arg-type] | |
| password=self.settings.hwm_store.password, # type: ignore[arg-type] | |
| ), | |
| namespace=self.settings.hwm_store.namespace, # type: ignore[arg-type] | |
| ).force_create_namespace() | |
| def _perform_incremental_transfer(self) -> None: | |
| with self._get_hwm_store() as hwm_store, IncrementalStrategy(): | |
| hwm_name = self._get_transfer_hwm_name() | |
| hwm = hwm_store.get_hwm(hwm_name) | |
| self.source_handler.hwm = hwm | |
| self.target_handler.hwm = hwm | |
| df = self.source_handler.read() | |
| self.target_handler.write(df) | |
| def _get_transfer_hwm_name(self) -> str: | |
| if isinstance(self.source_handler.transfer_dto, FileTransferDTO): | |
| hwm_name_suffix = self.source_handler.transfer_dto.directory_path | |
| else: | |
| hwm_name_suffix = self.source_handler.transfer_dto.table_name | |
| hwm_name = "_".join( | |
| [ | |
| str(self.source_handler.transfer_dto.id), | |
| self.source_handler.connection_dto.type, | |
| hwm_name_suffix, | |
| ], | |
| ) | |
| return hwm_name # noqa: RET504 | |
| def _reset_transfer_hwm(self) -> None: | |
| with self._get_hwm_store() as hwm_store: | |
| hwm_name = self._get_transfer_hwm_name() | |
| hwm = hwm_store.get_hwm(hwm_name) | |
| if hwm and hwm.value: | |
| hwm.reset() | |
| hwm_store.set_hwm(hwm) | |
| logger.warning( | |
| "HWM value has been reset to its default for transfer with id = %r", | |
| self.source_handler.transfer_dto.id, | |
| ) | |
Xet Storage Details
- Size:
- 10.5 kB
- Xet hash:
- ae7871dd9e4bc2c5ffaa54c70080b47e1402c314b82ef94485a05757a3a6363d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.