Spaces:

nenae18
/

leechard

Sleeping

Restore latest code (multi-code) after rollback isolation test

5bf61fe verified 4 months ago

6.73 kB

"""Enqueue 디스패치 + 실패 복구 (next_retry_at 게이트 존중).
  - relay_outbox: 발행 대상 outbox 이벤트를 enqueue. next_retry_at 과 task 적격성을 확인.
  - repair_stuck_queued: 직접 enqueue 하지 않는다. QUEUED 인데 미발행 outbox 가 없는 task 에
    outbox 이벤트만 보충(next_retry_at 승계). 실제 dispatch 는 relay_outbox 가 담당.
"""
from __future__ import annotations

import logging
from collections.abc import Callable
from datetime import datetime, timezone
from typing import Any

from sqlalchemy import update
from sqlalchemy.orm import Session

from app.domain.states import TaskStatus
from app.models.outbox import OutboxEvent
from app.models.photo_task import PhotoTask
from app.services import task_repo

logger = logging.getLogger(__name__)

EnqueueFn = Callable[[str, int], Any]


def _utcnow() -> datetime:
    return datetime.now(timezone.utc)


def _aware(dt: datetime | None) -> datetime | None:
    """sqlite 는 tz 를 잃은 naive 를 돌려준다 — UTC 로 간주해 비교 안전성 확보."""
    if dt is not None and dt.tzinfo is None:
        return dt.replace(tzinfo=timezone.utc)
    return dt


def default_enqueue(task_id: str, dispatch_version: int = 0) -> Any:
    """실제 Celery enqueue. payload 에 dispatch_version 을 실어 stale dispatch 를 차단."""
    from app.workers.tasks import process_photo

    return process_photo.delay(task_id, dispatch_version)


def _job_id(async_result: Any) -> str | None:
    return getattr(async_result, "id", None)


def mark_published(db: Session, task_id: str) -> None:
    """해당 task 의 미발행 outbox 이벤트를 발행 완료로 마킹(upload 성공 직후)."""
    now = _utcnow()
    events = (
        db.query(OutboxEvent)
        .filter(OutboxEvent.task_id == task_id, OutboxEvent.published.is_(False))
        .all()
    )
    for event in events:
        event.published = True
        event.status = "PUBLISHED"
        event.published_at = now
        event.attempts += 1
    if events:
        db.commit()


def _task_dispatchable(task: PhotoTask | None, now: datetime) -> bool:
    """photo_task 가 지금 dispatch 가능한 상태인지(재시도 게이트 포함)."""
    if task is None:
        return False
    if task.status != TaskStatus.QUEUED:
        return False
    if task.dead_lettered_at is not None:
        return False
    locked_until = _aware(task.locked_until)
    if locked_until is not None and locked_until >= now:
        return False
    next_retry_at = _aware(task.next_retry_at)
    if next_retry_at is not None and next_retry_at > now:
        return False
    return True


def relay_outbox(
    db: Session,
    *,
    enqueue: EnqueueFn = default_enqueue,
    limit: int = 100,
    now: datetime | None = None,
) -> int:
    """미발행 outbox 이벤트를 enqueue. 게이트 미통과 이벤트는 그대로 남긴다.
    이벤트 게이트: published=False AND (next_retry_at IS NULL OR <= now)
    task 게이트:  _task_dispatchable() (QUEUED, dead-letter 아님, lock 만료, next_retry 도달)
    """
    now = now or _utcnow()
    events = (
        db.query(OutboxEvent)
        .filter(
            OutboxEvent.status.in_(("PENDING", "FAILED")),
            OutboxEvent.published.is_(False),
            (OutboxEvent.next_retry_at.is_(None))
            | (OutboxEvent.next_retry_at <= now),
        )
        .order_by(OutboxEvent.created_at)
        .limit(limit)
        .all()
    )
    # 1) 적격성 판정. 읽기 트랜잭션은 닫는다(enqueue 는 브로커 I/O — DB 락 미보유).
    #    publish 직전 task 의 현재 dispatch_version 을 재확인:
    #    event.dispatch_version != task.dispatch_version → stale → STALE 로 마킹 후 skip.
    eligible = []
    stale_ids = []
    for ev in events:
        task = db.get(PhotoTask, ev.task_id)
        if task is not None and ev.dispatch_version != task.dispatch_version:
            stale_ids.append(ev.id)
            logger.info(
                "stale_dispatch_ignored: event=%s task=%s event_dv=%s task_dv=%s",
                ev.id, ev.task_id, ev.dispatch_version, task.dispatch_version,
            )
            continue
        if _task_dispatchable(task, now):
            eligible.append((ev.id, ev.task_id, ev.attempts, task.dispatch_version))
    for ev_id in stale_ids:
        db.execute(
            update(OutboxEvent)
            .where(OutboxEvent.id == ev_id)
            .values(status="STALE", last_error="stale_dispatch_ignored")
            .execution_options(synchronize_session=False)
        )
    db.commit()

    # 2) enqueue(payload 에 dispatch_version) + 짧은 트랜잭션으로 published 마킹.
    published = 0
    for ev_id, task_id, attempts, dversion in eligible:
        async_result = enqueue(task_id, dversion)
        db.execute(
            update(OutboxEvent)
            .where(OutboxEvent.id == ev_id)
            .values(
                published=True,
                status="PUBLISHED",
                published_at=_utcnow(),
                attempts=attempts + 1,
            )
            .execution_options(synchronize_session=False)
        )
        db.execute(
            update(PhotoTask)
            .where(
                PhotoTask.task_id == task_id,
                PhotoTask.celery_job_id.is_(None),
            )
            .values(celery_job_id=_job_id(async_result))
            .execution_options(synchronize_session=False)
        )
        db.commit()
        published += 1
    return published


def repair_stuck_queued(db: Session, *, limit: int = 100, now: datetime | None = None) -> int:
    """dispatch 가능한데 현재 dispatch_version 의 outbox 가 없는 task 에 이벤트 보충.
    직접 enqueue 하지 않는다. 멱등 helper(ensure_dispatch_event, rearm_published=False)로
    같은 dispatch_version 중복 생성을 막는다. next_retry_at 은 task 값을 승계.
    반환: 새로 만든(=created) outbox 이벤트 수.
    """
    now = now or _utcnow()
    tasks = (
        db.query(PhotoTask)
        .filter(
            PhotoTask.status == TaskStatus.QUEUED,
            PhotoTask.dead_lettered_at.is_(None),
        )
        .limit(limit)
        .all()
    )
    created = 0
    for task in tasks:
        # 게이트: lock 만료 + next_retry 도달
        if not _task_dispatchable(task, now):
            continue
        outcome = task_repo.ensure_dispatch_event(
            db,
            task.task_id,
            task.dispatch_version,
            next_retry_at=task.next_retry_at,
            rearm_published=False,
        )
        if outcome == "created":
            created += 1
    db.commit()
    return created

	"""Enqueue 디스패치 + 실패 복구 (next_retry_at 게이트 존중).

	- relay_outbox: 발행 대상 outbox 이벤트를 enqueue. next_retry_at 과 task 적격성을 확인.
	- repair_stuck_queued: 직접 enqueue 하지 않는다. QUEUED 인데 미발행 outbox 가 없는 task 에
	outbox 이벤트만 보충(next_retry_at 승계). 실제 dispatch 는 relay_outbox 가 담당.
	"""
	from __future__ import annotations

	import logging
	from collections.abc import Callable
	from datetime import datetime, timezone
	from typing import Any

	from sqlalchemy import update
	from sqlalchemy.orm import Session

	from app.domain.states import TaskStatus
	from app.models.outbox import OutboxEvent
	from app.models.photo_task import PhotoTask
	from app.services import task_repo

	logger = logging.getLogger(__name__)

	EnqueueFn = Callable[[str, int], Any]


	def _utcnow() -> datetime:
	return datetime.now(timezone.utc)


	def _aware(dt: datetime \| None) -> datetime \| None:
	"""sqlite 는 tz 를 잃은 naive 를 돌려준다 — UTC 로 간주해 비교 안전성 확보."""
	if dt is not None and dt.tzinfo is None:
	return dt.replace(tzinfo=timezone.utc)
	return dt


	def default_enqueue(task_id: str, dispatch_version: int = 0) -> Any:
	"""실제 Celery enqueue. payload 에 dispatch_version 을 실어 stale dispatch 를 차단."""
	from app.workers.tasks import process_photo

	return process_photo.delay(task_id, dispatch_version)


	def _job_id(async_result: Any) -> str \| None:
	return getattr(async_result, "id", None)


	def mark_published(db: Session, task_id: str) -> None:
	"""해당 task 의 미발행 outbox 이벤트를 발행 완료로 마킹(upload 성공 직후)."""
	now = _utcnow()
	events = (
	db.query(OutboxEvent)
	.filter(OutboxEvent.task_id == task_id, OutboxEvent.published.is_(False))
	.all()
	)
	for event in events:
	event.published = True
	event.status = "PUBLISHED"
	event.published_at = now
	event.attempts += 1
	if events:
	db.commit()


	def _task_dispatchable(task: PhotoTask \| None, now: datetime) -> bool:
	"""photo_task 가 지금 dispatch 가능한 상태인지(재시도 게이트 포함)."""
	if task is None:
	return False
	if task.status != TaskStatus.QUEUED:
	return False
	if task.dead_lettered_at is not None:
	return False
	locked_until = _aware(task.locked_until)
	if locked_until is not None and locked_until >= now:
	return False
	next_retry_at = _aware(task.next_retry_at)
	if next_retry_at is not None and next_retry_at > now:
	return False
	return True


	def relay_outbox(
	db: Session,
	*,
	enqueue: EnqueueFn = default_enqueue,
	limit: int = 100,
	now: datetime \| None = None,
	) -> int:
	"""미발행 outbox 이벤트를 enqueue. 게이트 미통과 이벤트는 그대로 남긴다.

	이벤트 게이트: published=False AND (next_retry_at IS NULL OR <= now)
	task 게이트: _task_dispatchable() (QUEUED, dead-letter 아님, lock 만료, next_retry 도달)
	"""
	now = now or _utcnow()
	events = (
	db.query(OutboxEvent)
	.filter(
	OutboxEvent.status.in_(("PENDING", "FAILED")),
	OutboxEvent.published.is_(False),
	(OutboxEvent.next_retry_at.is_(None))
	\| (OutboxEvent.next_retry_at <= now),
	)
	.order_by(OutboxEvent.created_at)
	.limit(limit)
	.all()
	)
	# 1) 적격성 판정. 읽기 트랜잭션은 닫는다(enqueue 는 브로커 I/O — DB 락 미보유).
	# publish 직전 task 의 현재 dispatch_version 을 재확인:
	# event.dispatch_version != task.dispatch_version → stale → STALE 로 마킹 후 skip.
	eligible = []
	stale_ids = []
	for ev in events:
	task = db.get(PhotoTask, ev.task_id)
	if task is not None and ev.dispatch_version != task.dispatch_version:
	stale_ids.append(ev.id)
	logger.info(
	"stale_dispatch_ignored: event=%s task=%s event_dv=%s task_dv=%s",
	ev.id, ev.task_id, ev.dispatch_version, task.dispatch_version,
	)
	continue
	if _task_dispatchable(task, now):
	eligible.append((ev.id, ev.task_id, ev.attempts, task.dispatch_version))
	for ev_id in stale_ids:
	db.execute(
	update(OutboxEvent)
	.where(OutboxEvent.id == ev_id)
	.values(status="STALE", last_error="stale_dispatch_ignored")
	.execution_options(synchronize_session=False)
	)
	db.commit()

	# 2) enqueue(payload 에 dispatch_version) + 짧은 트랜잭션으로 published 마킹.
	published = 0
	for ev_id, task_id, attempts, dversion in eligible:
	async_result = enqueue(task_id, dversion)
	db.execute(
	update(OutboxEvent)
	.where(OutboxEvent.id == ev_id)
	.values(
	published=True,
	status="PUBLISHED",
	published_at=_utcnow(),
	attempts=attempts + 1,
	)
	.execution_options(synchronize_session=False)
	)
	db.execute(
	update(PhotoTask)
	.where(
	PhotoTask.task_id == task_id,
	PhotoTask.celery_job_id.is_(None),
	)
	.values(celery_job_id=_job_id(async_result))
	.execution_options(synchronize_session=False)
	)
	db.commit()
	published += 1
	return published


	def repair_stuck_queued(db: Session, *, limit: int = 100, now: datetime \| None = None) -> int:
	"""dispatch 가능한데 현재 dispatch_version 의 outbox 가 없는 task 에 이벤트 보충.

	직접 enqueue 하지 않는다. 멱등 helper(ensure_dispatch_event, rearm_published=False)로
	같은 dispatch_version 중복 생성을 막는다. next_retry_at 은 task 값을 승계.
	반환: 새로 만든(=created) outbox 이벤트 수.
	"""
	now = now or _utcnow()
	tasks = (
	db.query(PhotoTask)
	.filter(
	PhotoTask.status == TaskStatus.QUEUED,
	PhotoTask.dead_lettered_at.is_(None),
	)
	.limit(limit)
	.all()
	)
	created = 0
	for task in tasks:
	# 게이트: lock 만료 + next_retry 도달
	if not _task_dispatchable(task, now):
	continue
	outcome = task_repo.ensure_dispatch_event(
	db,
	task.task_id,
	task.dispatch_version,
	next_retry_at=task.next_retry_at,
	rearm_published=False,
	)
	if outcome == "created":
	created += 1
	db.commit()
	return created