Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /MTSWebServices_syncmaster /syncmaster /worker /transfer.py
| # SPDX-FileCopyrightText: 2023-present MTS PJSC | |
| # SPDX-License-Identifier: Apache-2.0 | |
| from datetime import UTC, datetime | |
| from asgi_correlation_id import correlation_id | |
| from celery.signals import after_setup_task_logger, before_task_publish, task_postrun, task_prerun | |
| from celery.utils.log import get_task_logger | |
| from jinja2 import Template | |
| from sqlalchemy import Engine, select | |
| from sqlalchemy.orm import Session, selectinload | |
| from uuid6 import uuid8 | |
| from syncmaster.db.models import AuthData, Run, Status, Transfer | |
| from syncmaster.db.repositories.utils import decrypt_auth_data | |
| from syncmaster.settings.logging import setup_logging | |
| from syncmaster.worker.base import WorkerTask | |
| from syncmaster.worker.celery import app as celery | |
| from syncmaster.worker.controller import TransferController | |
| from syncmaster.worker.settings import WorkerAppSettings | |
| logger = get_task_logger(__name__) | |
| CORRELATION_ID_HEADER = "CORRELATION_ID" | |
| def transfer_correlation_id(headers: dict[str, str], **kwargs) -> None: | |
| """Transfer correlation ID from the request to the Celery task headers.""" | |
| cid = correlation_id.get() | |
| if cid: | |
| headers[CORRELATION_ID_HEADER] = cid | |
| def load_correlation_id(task, **kwargs) -> None: | |
| """Load correlation ID from task headers, or generate a new one.""" | |
| id_value = task.request.get(CORRELATION_ID_HEADER) | |
| if id_value: | |
| correlation_id.set(id_value) | |
| else: | |
| correlation_id.set(uuid8().hex) | |
| def cleanup_correlation_id(**kwargs) -> None: | |
| """Clear the correlation ID after the task completes.""" | |
| correlation_id.set(None) | |
| def run_transfer_task(self: WorkerTask, run_id: int) -> None: | |
| run_transfer( | |
| run_id=run_id, | |
| engine=self.engine, | |
| settings=self.settings, | |
| ) | |
| def run_transfer(run_id: int, engine: Engine, settings: WorkerAppSettings): | |
| exception: Exception | None = None | |
| try: | |
| with Session(engine, expire_on_commit=False) as session: | |
| run = session.scalar( | |
| select(Run) | |
| .where( | |
| Run.id == run_id, | |
| Run.status == Status.CREATED, | |
| ) | |
| .options( | |
| selectinload(Run.transfer), | |
| selectinload(Run.transfer).selectinload(Transfer.group), | |
| selectinload(Run.transfer).selectinload(Transfer.source_connection), | |
| selectinload(Run.transfer).selectinload(Transfer.target_connection), | |
| ), | |
| ) | |
| if run is None: | |
| logger.info("Run %r not found or already executed by someone else", run_id) | |
| return | |
| source_connection = run.transfer.source_connection | |
| source_auth_data_query = select(AuthData).where(AuthData.connection_id == run.transfer.source_connection.id) | |
| source_auth_data_encrypted = session.scalars(source_auth_data_query).one() | |
| source_auth_data = decrypt_auth_data(source_auth_data_encrypted.value, settings) | |
| target_connection = run.transfer.target_connection | |
| target_auth_data_query = select(AuthData).where(AuthData.connection_id == run.transfer.target_connection.id) | |
| target_auth_data_encrypted = session.scalars(target_auth_data_query).one() | |
| target_auth_data = decrypt_auth_data(target_auth_data_encrypted.value, settings) | |
| logger.info("Starting run %r", run_id) | |
| run.status = Status.STARTED | |
| run.started_at = datetime.now(tz=UTC) | |
| run.log_url = Template(settings.worker.log_url_template).render( | |
| run=run, | |
| correlation_id=correlation_id.get(), | |
| ) | |
| session.commit() | |
| session.expunge_all() # do not hold open DB connection while run is executed | |
| controller = TransferController( | |
| settings=settings, | |
| run=run, | |
| source_connection=source_connection, | |
| target_connection=target_connection, | |
| source_auth_data=source_auth_data, | |
| target_auth_data=target_auth_data, | |
| ) | |
| controller.perform_transfer() | |
| except Exception as e: | |
| status = Status.FAILED | |
| logger.exception("Run %r failed", run_id) | |
| exception = e | |
| else: | |
| status = Status.FINISHED | |
| logger.info("Run %r was successful", run_id) | |
| with Session(engine) as session: | |
| run = session.get(Run, run_id) | |
| if run: | |
| logger.info("Updating run %r status in DB", run_id) | |
| run.status = status | |
| run.ended_at = datetime.now(tz=UTC) | |
| session.commit() | |
| if exception is not None: | |
| raise exception | |
| def setup_loggers(*args, **kwargs): | |
| setup_logging(WorkerAppSettings().logging) | |
Xet Storage Details
- Size:
- 4.98 kB
- Xet hash:
- 67593c13454488de5c2c7d2491d26c497af52e45b4fc94f5e6a19751ea6f718e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.