bimquery-api / app /core /celery_app.py
BIM Query Deploy
Claude Opus 4.7 (1M context)
deploy: initial BIM Query API on Hugging Face Spaces
e6be672
Raw History Blame Contribute Delete
3.92 kB
# Copyright (C) 2026 Mattia Zatelli
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Celery application configuration
"""
import logging
from celery import Celery
from celery.signals import setup_logging
from app.core.config import get_settings
settings = get_settings()
# Create Celery app
celery_app = Celery(
"bim-query",
broker=settings.REDIS_URL,
backend=settings.REDIS_URL,
include=[
"app.tasks.ifc_processing",
"app.tasks.cleanup",
]
)
"""
Celery configuration for background tasks
This module configures Celery for asynchronous processing:
- Background IFC file processing
- Embedding generation for semantic search
- Automatic cleanup of temporary files
- Scheduled batch processing
- Tiered storage management
Uses Redis as message broker with intelligent task routing
to specialized queues for performance optimization.
"""
celery_app.conf.update(
# Task routing
task_routes={
"app.tasks.ifc_processing.process_ifc_file": {"queue": "ifc_processing"},
"app.tasks.ifc_processing.enrich_geometry": {"queue": "ifc_processing"},
"app.tasks.cleanup.cleanup_expired_exports": {"queue": "cleanup"},
"*.dlq": {"queue": "dlq"},
},
# Task settings
task_serializer="json",
result_serializer="json",
accept_content=["json"],
result_expires=3600,
timezone="UTC",
enable_utc=True,
# Worker settings
worker_prefetch_multiplier=1,
task_acks_late=True,
worker_max_tasks_per_child=1000,
# Retry settings
task_reject_on_worker_lost=True,
task_acks_on_failure_or_timeout=True,
# Rate limiting
task_default_rate_limit="10/m",
# Beat schedule for periodic tasks
beat_schedule={
"cleanup-expired-exports": {
"task": "app.tasks.cleanup.cleanup_expired_exports",
"schedule": 3600.0, # Every hour
},
"process-ifc-stream": {
"task": "app.tasks.ifc_processing.process_ifc_stream",
"schedule": 10.0, # Every 10 seconds
},
}
)
# Configure logging
@setup_logging.connect
def config_loggers(*args, **kwargs):
"""Configure Celery logging to use app logging configuration"""
from logging.config import dictConfig
dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'default': {
'format': '[%(asctime)s: %(levelname)s/%(processName)s] %(name)s: %(message)s',
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'default',
},
'file': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': 'logs/celery.log',
'formatter': 'default',
},
},
'root': {
'level': 'INFO',
'handlers': ['console', 'file'],
},
'loggers': {
'celery.task': {
'level': 'INFO',
'handlers': ['console', 'file'],
'propagate': False,
},
},
})
# Auto-discover tasks
celery_app.autodiscover_tasks()
if __name__ == "__main__":
celery_app.start()