# Copyright (C) 2026 Mattia Zatelli # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . """ Celery application configuration """ import logging from celery import Celery from celery.signals import setup_logging from app.core.config import get_settings settings = get_settings() # Create Celery app celery_app = Celery( "bim-query", broker=settings.REDIS_URL, backend=settings.REDIS_URL, include=[ "app.tasks.ifc_processing", "app.tasks.cleanup", ] ) """ Celery configuration for background tasks This module configures Celery for asynchronous processing: - Background IFC file processing - Embedding generation for semantic search - Automatic cleanup of temporary files - Scheduled batch processing - Tiered storage management Uses Redis as message broker with intelligent task routing to specialized queues for performance optimization. """ celery_app.conf.update( # Task routing task_routes={ "app.tasks.ifc_processing.process_ifc_file": {"queue": "ifc_processing"}, "app.tasks.ifc_processing.enrich_geometry": {"queue": "ifc_processing"}, "app.tasks.cleanup.cleanup_expired_exports": {"queue": "cleanup"}, "*.dlq": {"queue": "dlq"}, }, # Task settings task_serializer="json", result_serializer="json", accept_content=["json"], result_expires=3600, timezone="UTC", enable_utc=True, # Worker settings worker_prefetch_multiplier=1, task_acks_late=True, worker_max_tasks_per_child=1000, # Retry settings task_reject_on_worker_lost=True, task_acks_on_failure_or_timeout=True, # Rate limiting task_default_rate_limit="10/m", # Beat schedule for periodic tasks beat_schedule={ "cleanup-expired-exports": { "task": "app.tasks.cleanup.cleanup_expired_exports", "schedule": 3600.0, # Every hour }, "process-ifc-stream": { "task": "app.tasks.ifc_processing.process_ifc_stream", "schedule": 10.0, # Every 10 seconds }, } ) # Configure logging @setup_logging.connect def config_loggers(*args, **kwargs): """Configure Celery logging to use app logging configuration""" from logging.config import dictConfig dictConfig({ 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'default': { 'format': '[%(asctime)s: %(levelname)s/%(processName)s] %(name)s: %(message)s', }, }, 'handlers': { 'console': { 'level': 'INFO', 'class': 'logging.StreamHandler', 'formatter': 'default', }, 'file': { 'level': 'INFO', 'class': 'logging.FileHandler', 'filename': 'logs/celery.log', 'formatter': 'default', }, }, 'root': { 'level': 'INFO', 'handlers': ['console', 'file'], }, 'loggers': { 'celery.task': { 'level': 'INFO', 'handlers': ['console', 'file'], 'propagate': False, }, }, }) # Auto-discover tasks celery_app.autodiscover_tasks() if __name__ == "__main__": celery_app.start()