Buckets:

download
raw
10.5 kB
# SPDX-FileCopyrightText: 2023-present MTS PJSC
# SPDX-License-Identifier: Apache-2.0
import logging
from collections.abc import Callable
from tempfile import TemporaryDirectory
from typing import Any
from etl_entities.hwm_store import BaseHWMStore
from horizon.client.auth import LoginPassword
from horizon_hwm_store import HorizonHWMStore
from onetl.hooks import slot, support_hooks
from onetl.strategy import IncrementalStrategy
from syncmaster.db.models import Connection, Run
from syncmaster.dto.connections import (
ClickhouseConnectionDTO,
ConnectionDTO,
FTPConnectionDTO,
FTPSConnectionDTO,
HDFSConnectionDTO,
HiveConnectionDTO,
MSSQLConnectionDTO,
MySQLConnectionDTO,
OracleConnectionDTO,
PostgresConnectionDTO,
S3ConnectionDTO,
SambaConnectionDTO,
SFTPConnectionDTO,
WebDAVConnectionDTO,
get_iceberg_connection_dto,
)
from syncmaster.dto.runs import RunDTO
from syncmaster.dto.transfers import (
ClickhouseTransferDTO,
DBTransferDTO,
FileTransferDTO,
FTPSTransferDTO,
FTPTransferDTO,
HDFSTransferDTO,
HiveTransferDTO,
IcebergTransferDTO,
MSSQLTransferDTO,
MySQLTransferDTO,
OracleTransferDTO,
PostgresTransferDTO,
S3TransferDTO,
SambaTransferDTO,
SFTPTransferDTO,
WebDAVTransferDTO,
)
from syncmaster.dto.transfers_resources import Resources
from syncmaster.dto.transfers_strategy import Strategy
from syncmaster.exceptions.connection import ConnectionTypeNotRecognizedError
from syncmaster.worker.handlers.base import Handler
from syncmaster.worker.handlers.db.clickhouse import ClickhouseHandler
from syncmaster.worker.handlers.db.hive import HiveHandler
from syncmaster.worker.handlers.db.iceberg import IcebergRESTCatalogS3Handler
from syncmaster.worker.handlers.db.mssql import MSSQLHandler
from syncmaster.worker.handlers.db.mysql import MySQLHandler
from syncmaster.worker.handlers.db.oracle import OracleHandler
from syncmaster.worker.handlers.db.postgres import PostgresHandler
from syncmaster.worker.handlers.file.ftp import FTPHandler
from syncmaster.worker.handlers.file.ftps import FTPSHandler
from syncmaster.worker.handlers.file.hdfs import HDFSHandler
from syncmaster.worker.handlers.file.s3 import S3Handler
from syncmaster.worker.handlers.file.samba import SambaHandler
from syncmaster.worker.handlers.file.sftp import SFTPHandler
from syncmaster.worker.handlers.file.webdav import WebDAVHandler
from syncmaster.worker.settings import WorkerAppSettings
logger = logging.getLogger(__name__)
HANDLERS_MAPPING = dict[
str,
tuple[
type[Handler],
Callable[..., ConnectionDTO],
type[DBTransferDTO] | type[FileTransferDTO],
type[RunDTO],
],
]
connection_handler_proxy: HANDLERS_MAPPING = {
"hive": (
HiveHandler,
HiveConnectionDTO,
HiveTransferDTO,
RunDTO,
),
"iceberg": (
IcebergRESTCatalogS3Handler,
get_iceberg_connection_dto,
IcebergTransferDTO,
RunDTO,
),
"oracle": (
OracleHandler,
OracleConnectionDTO,
OracleTransferDTO,
RunDTO,
),
"clickhouse": (
ClickhouseHandler,
ClickhouseConnectionDTO,
ClickhouseTransferDTO,
RunDTO,
),
"mssql": (
MSSQLHandler,
MSSQLConnectionDTO,
MSSQLTransferDTO,
RunDTO,
),
"mysql": (
MySQLHandler,
MySQLConnectionDTO,
MySQLTransferDTO,
RunDTO,
),
"postgres": (
PostgresHandler,
PostgresConnectionDTO,
PostgresTransferDTO,
RunDTO,
),
"s3": (
S3Handler,
S3ConnectionDTO,
S3TransferDTO,
RunDTO,
),
"hdfs": (
HDFSHandler,
HDFSConnectionDTO,
HDFSTransferDTO,
RunDTO,
),
"sftp": (
SFTPHandler,
SFTPConnectionDTO,
SFTPTransferDTO,
RunDTO,
),
"ftp": (
FTPHandler,
FTPConnectionDTO,
FTPTransferDTO,
RunDTO,
),
"ftps": (
FTPSHandler,
FTPSConnectionDTO,
FTPSTransferDTO,
RunDTO,
),
"samba": (
SambaHandler,
SambaConnectionDTO,
SambaTransferDTO,
RunDTO,
),
"webdav": (
WebDAVHandler,
WebDAVConnectionDTO,
WebDAVTransferDTO,
RunDTO,
),
}
@support_hooks
class TransferController:
settings: WorkerAppSettings
source_handler: Handler
target_handler: Handler
def __init__( # noqa: PLR0913, PLR0917
self,
settings: WorkerAppSettings,
run: Run,
source_connection: Connection,
source_auth_data: dict,
target_connection: Connection,
target_auth_data: dict,
):
self.temp_dir = TemporaryDirectory(prefix=f"syncmaster_{run.id}_")
self.settings = settings
self.run = run
self.source_handler = self.get_handler(
connection_data=source_connection.data,
run_data={"id": run.id, "created_at": run.created_at},
transfer_id=run.transfer.id,
transfer_name=run.transfer.name,
group_name=run.transfer.group.name,
transfer_params=run.transfer.source_params,
strategy_params=run.transfer.strategy_params,
resources=run.transfer.resources,
transformations=run.transfer.transformations,
connection_auth_data=source_auth_data,
temp_dir=TemporaryDirectory(dir=self.temp_dir.name, prefix="downloaded_"),
)
self.target_handler = self.get_handler(
connection_data=target_connection.data,
run_data={"id": run.id, "created_at": run.created_at},
transfer_id=run.transfer.id,
transfer_name=run.transfer.name,
group_name=run.transfer.group.name,
transfer_params=run.transfer.target_params,
strategy_params=run.transfer.strategy_params,
resources=run.transfer.resources,
transformations=run.transfer.transformations,
connection_auth_data=target_auth_data,
temp_dir=TemporaryDirectory(dir=self.temp_dir.name, prefix="written_"),
)
@slot
def perform_transfer(self) -> None:
try:
spark = self.settings.worker.create_spark_session_function(
run=self.run,
source=self.source_handler.connection_dto,
target=self.target_handler.connection_dto,
settings=self.settings.worker,
)
with spark:
self.source_handler.connect(spark)
self.target_handler.connect(spark)
if self.source_handler.transfer_dto.strategy.type == "incremental":
return self._perform_incremental_transfer()
if self.source_handler.transfer_dto.strategy.type == "full" and self.settings.hwm_store.enabled:
self._reset_transfer_hwm()
df = self.source_handler.read()
self.target_handler.write(df)
finally:
self.temp_dir.cleanup()
def get_handler( # noqa: PLR0913, PLR0917
self,
connection_data: dict[str, Any],
connection_auth_data: dict,
run_data: dict[str, Any],
transfer_id: int,
transfer_name: str,
group_name: str,
transfer_params: dict[str, Any],
strategy_params: dict[str, Any],
resources: dict[str, Any],
transformations: list[dict],
temp_dir: TemporaryDirectory,
) -> Handler:
connection_data.update(connection_auth_data)
connection_data.pop("type")
handler_type = transfer_params.pop("type", None)
if connection_handler_proxy.get(handler_type, None) is None:
raise ConnectionTypeNotRecognizedError
handler, connection_dto, transfer_dto, run_dto = connection_handler_proxy[handler_type]
return handler(
connection_dto=connection_dto(**connection_data),
transfer_dto=transfer_dto(
id=transfer_id,
name=transfer_name,
group_name=group_name,
strategy=Strategy.from_dict(strategy_params),
resources=Resources(**resources),
transformations=transformations,
**transfer_params,
),
run_dto=run_dto(**run_data),
temp_dir=temp_dir,
)
def _get_hwm_store(self) -> BaseHWMStore:
return HorizonHWMStore(
api_url=self.settings.hwm_store.url, # type: ignore[arg-type]
auth=LoginPassword(
login=self.settings.hwm_store.user, # type: ignore[arg-type]
password=self.settings.hwm_store.password, # type: ignore[arg-type]
),
namespace=self.settings.hwm_store.namespace, # type: ignore[arg-type]
).force_create_namespace()
def _perform_incremental_transfer(self) -> None:
with self._get_hwm_store() as hwm_store, IncrementalStrategy():
hwm_name = self._get_transfer_hwm_name()
hwm = hwm_store.get_hwm(hwm_name)
self.source_handler.hwm = hwm
self.target_handler.hwm = hwm
df = self.source_handler.read()
self.target_handler.write(df)
def _get_transfer_hwm_name(self) -> str:
if isinstance(self.source_handler.transfer_dto, FileTransferDTO):
hwm_name_suffix = self.source_handler.transfer_dto.directory_path
else:
hwm_name_suffix = self.source_handler.transfer_dto.table_name
hwm_name = "_".join(
[
str(self.source_handler.transfer_dto.id),
self.source_handler.connection_dto.type,
hwm_name_suffix,
],
)
return hwm_name # noqa: RET504
def _reset_transfer_hwm(self) -> None:
with self._get_hwm_store() as hwm_store:
hwm_name = self._get_transfer_hwm_name()
hwm = hwm_store.get_hwm(hwm_name)
if hwm and hwm.value:
hwm.reset()
hwm_store.set_hwm(hwm)
logger.warning(
"HWM value has been reset to its default for transfer with id = %r",
self.source_handler.transfer_dto.id,
)

Xet Storage Details

Size:
10.5 kB
·
Xet hash:
ae7871dd9e4bc2c5ffaa54c70080b47e1402c314b82ef94485a05757a3a6363d

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.