# Copyright (C) 2026 Mattia Zatelli # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . """ IFC processing Celery tasks """ import asyncio import json as _json import logging import os import traceback from datetime import datetime from typing import Dict, Any from celery import current_task from sqlalchemy import select from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession from app.core.celery_app import celery_app from app.core.config import get_settings from app.core.s3 import S3StorageService from app.core.redis import RedisStream from app.models import Project, ProjectStatus, Element from app.services.ifc_processor import IFCProcessor logger = logging.getLogger(__name__) settings = get_settings() # Don't create engine globally - create per task to avoid event loop issues def get_async_session_maker(): """Create async session maker per task to avoid event loop conflicts""" engine = create_async_engine( settings.DATABASE_URL, pool_pre_ping=True, pool_recycle=3600 ) return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) @celery_app.task( bind=True, max_retries=3, default_retry_delay=60, acks_late=True, reject_on_worker_lost=True, time_limit=7200, # 2 hours hard limit for large files soft_time_limit=6000 # 100 minutes soft limit ) def process_ifc_file(self, project_data: Dict[str, Any]): """ Process uploaded IFC file asynchronously Args: project_data: Dictionary containing project information from upload """ # Validate required task parameters required_fields = ("project_id", "tenant_id", "s3_key", "filename") missing = [f for f in required_fields if not project_data.get(f)] if missing: logger.error(f"Rejecting task: missing required fields: {missing}") return {"status": "rejected", "error": f"Missing fields: {missing}"} project_id = project_data["project_id"] tenant_id = project_data["tenant_id"] s3_bucket = project_data.get("s3_bucket") s3_key = project_data["s3_key"] filename = project_data["filename"] # Validate project_id format if not project_id.startswith("prj_"): logger.error(f"Rejecting task: invalid project_id format: {project_id}") return {"status": "rejected", "error": "Invalid project_id format"} # Validate S3 key prefix to prevent path traversal if not s3_key.startswith("projects/") or ".." in s3_key: logger.error(f"Rejecting task: invalid s3_key: {s3_key}") return {"status": "rejected", "error": "Invalid s3_key"} logger.info(f"Starting IFC processing for project {project_id}") # Implement Redis lock to prevent duplicate task execution from app.core.redis import redis_client lock_key = f"ifc_processing_lock:{project_id}" # Try to acquire lock with 30 minute timeout lock = redis_client.lock(lock_key, timeout=1800, blocking=False) if not lock.acquire(blocking=False): logger.warning(f"Another task is already processing project {project_id}, skipping duplicate") return { "status": "skipped", "project_id": project_id, "message": "Another processing task is already running for this project" } try: # Update task progress current_task.update_state( state="PROGRESS", meta={"stage": "initializing", "percent": 0} ) # Run the async processing # In Celery worker, we should NOT have a running loop, so use asyncio.run try: asyncio.run(_process_ifc_async(project_data)) except RuntimeError as e: # If there's already a loop running, create a new one in a thread if "running event loop" in str(e): logger.warning("Detected running event loop, creating new loop in thread") import concurrent.futures def run_in_thread(): new_loop = asyncio.new_event_loop() asyncio.set_event_loop(new_loop) try: return new_loop.run_until_complete(_process_ifc_async(project_data)) finally: new_loop.close() asyncio.set_event_loop(None) with concurrent.futures.ThreadPoolExecutor() as executor: future = executor.submit(run_in_thread) future.result() else: raise logger.info(f"IFC processing completed for project {project_id}") return { "status": "completed", "project_id": project_id, "message": "IFC file processed successfully" } except Exception as exc: logger.error(f"IFC processing failed for project {project_id}: {exc}") logger.error(traceback.format_exc()) # Update project status to failed try: asyncio.run(_mark_project_failed(project_id, str(exc))) except RuntimeError as e: if "running event loop" in str(e): logger.warning("Detected running event loop when marking failed") import concurrent.futures def mark_failed_in_thread(): new_loop = asyncio.new_event_loop() asyncio.set_event_loop(new_loop) try: return new_loop.run_until_complete(_mark_project_failed(project_id, str(exc))) finally: new_loop.close() asyncio.set_event_loop(None) with concurrent.futures.ThreadPoolExecutor() as executor: future = executor.submit(mark_failed_in_thread) future.result() else: logger.error(f"Failed to mark project as failed: {e}") # Retry logic if self.request.retries < self.max_retries: logger.info(f"Retrying IFC processing for project {project_id} (attempt {self.request.retries + 1})") raise self.retry(exc=exc, countdown=60 * (self.request.retries + 1)) # Final failure return { "status": "failed", "project_id": project_id, "error": str(exc) } finally: # Always release the lock try: lock.release() logger.info(f"Released processing lock for project {project_id}") except Exception as e: logger.warning(f"Failed to release lock for project {project_id}: {e}") def _update_redis_progress(project_id: str, percent: int, stage: str): """Write real-time progress to Redis for the /status endpoint""" try: from app.core.redis import redis_client if redis_client: redis_client.set( f"project_progress:{project_id}", _json.dumps({"percent": percent, "stage": stage}), ex=600 ) except Exception: pass async def _process_ifc_async(project_data: Dict[str, Any]): """Async IFC processing logic""" project_id = project_data.get("project_id") s3_bucket = project_data.get("s3_bucket") s3_key = project_data.get("s3_key") filename = project_data.get("filename") logger.info(f"DEBUG: Starting async processing for project {project_id} with s3_key: {s3_key}") # Create session maker for this task instance async_session_maker = get_async_session_maker() async with async_session_maker() as session: # Get project record try: internal_id = int(project_id.replace('prj_', '')) except ValueError: raise ValueError(f"Invalid project ID format: {project_id}") query = select(Project).where(Project.id == internal_id) result = await session.execute(query) project = result.scalar_one_or_none() if not project: raise ValueError(f"Project not found: {project_id}") try: # Mark as processing project.mark_processing("downloading") await session.commit() # Update progress current_task.update_state( state="PROGRESS", meta={"stage": "downloading", "percent": 10} ) # Initialize S3 client if not already done from app.core.s3 import s3_client, init_s3 if not s3_client: logger.info("Initializing S3 client for IFC processing") init_s3() # Download file from S3 logger.info(f"DEBUG: Starting download for project {project_id} from S3 key: {s3_key}") file_content = S3StorageService.download_file_by_key(s3_key) logger.info(f"DEBUG: Download completed, file size: {len(file_content)} bytes") # Update progress current_task.update_state( state="PROGRESS", meta={"stage": "parsing", "percent": 20} ) # Initialize IFC processor processor = IFCProcessor() # Parse IFC file logger.info(f"DEBUG: Starting IFC parsing for project {project_id}") ifc_data = await processor.parse_ifc_content(file_content, filename) logger.info(f"DEBUG: IFC parsing completed, found {len(ifc_data.get('elements', []))} elements") # Update project metadata project.ifc_schema = ifc_data.get("schema") project.ifc_version = ifc_data.get("version") # Store project info with units project_info = ifc_data.get("project_info", {}) units = ifc_data.get("units", {}) logger.info(f"DEBUG: Extracted units for project {project_id}: {units}") project_info['units'] = units project.project_info = project_info # Force SQLAlchemy to detect the change in JSONB field from sqlalchemy.orm.attributes import flag_modified flag_modified(project, "project_info") logger.info(f"DEBUG: Set project_info with units for project {project_id}: {project.project_info}") project.site_info = ifc_data.get("site_info", {}) project.bounding_box = ifc_data.get("bounding_box") project.gross_floor_area = ifc_data.get("gross_floor_area") project.building_height = ifc_data.get("building_height") project.update_progress(40, "extracting_elements") await session.commit() logger.info(f"DEBUG: Committed project metadata for project {project_id}") # Update progress current_task.update_state( state="PROGRESS", meta={"stage": "extracting_elements", "percent": 40} ) # Extract elements with progress callback logger.info(f"Extracting elements for project {project_id}") def extraction_progress(current, total): """Callback for extraction progress updates (sync, called from thread pool)""" try: progress_percent = 40 + int(current / total * 20) current_task.update_state( state="PROGRESS", meta={"stage": "extracting_elements", "percent": progress_percent} ) _update_redis_progress(project_id, progress_percent, "extracting_elements") except Exception as e: logger.warning(f"Error in extraction progress callback: {e}") elements_data = await processor.extract_elements(ifc_data.get("ifc_model"), extraction_progress) # Persist final extraction progress to DB project.update_progress(60, "extraction_complete") await session.commit() logger.info(f"Extraction completed: {len(elements_data)} elements") # Update progress current_task.update_state( state="PROGRESS", meta={"stage": "storing_elements", "percent": 60} ) _update_redis_progress(project_id, 60, "storing_elements") # Store elements in database logger.info(f"DEBUG: Starting database storage of {len(elements_data)} elements for project {project_id}") await _store_elements(session, project, elements_data, current_task, project_id) logger.info(f"DEBUG: Database storage completed for project {project_id}") project.update_progress(80, "indexing") await session.commit() # Update progress current_task.update_state( state="PROGRESS", meta={"stage": "indexing", "percent": 80} ) # Create search indexes and embeddings await processor.create_search_indexes(project.id, elements_data) project.update_progress(95, "finalizing") current_task.update_state( state="PROGRESS", meta={"stage": "finalizing", "percent": 95} ) _update_redis_progress(project_id, 95, "finalizing") # Calculate final statistics in single pass element_count = len(elements_data) element_types = {} has_geometry = False has_properties = False has_materials = False has_spaces = False for elem in elements_data: ifc_type = elem.get("ifc_type", "Unknown") element_types[ifc_type] = element_types.get(ifc_type, 0) + 1 if not has_geometry and elem.get("has_geometry"): has_geometry = True if not has_properties and elem.get("properties"): has_properties = True if not has_materials and elem.get("material_name"): has_materials = True if not has_spaces and ifc_type == "IfcSpace": has_spaces = True # Update project final status project.element_count = element_count project.element_types = element_types project.has_geometry = has_geometry project.has_properties = has_properties project.has_materials = has_materials project.has_spaces = has_spaces project.mark_ready() await session.commit() # Final update current_task.update_state( state="SUCCESS", meta={"stage": "ready", "percent": 100} ) _update_redis_progress(project_id, 100, "ready") logger.info(f"IFC processing completed for project {project_id}: {element_count} elements") # Schedule background geometry enrichment (non-blocking) try: enrich_geometry.delay({ "project_id": project_id, "internal_id": internal_id, "s3_key": s3_key, "filename": filename, }) logger.info(f"Scheduled geometry enrichment for project {project_id}") except Exception as e: logger.warning(f"Could not schedule geometry enrichment: {e}") except Exception as e: project.mark_failed(str(e), {"traceback": traceback.format_exc()}) await session.commit() raise async def _store_elements( session: AsyncSession, project: Project, elements_data: list, task=None, project_id: str = None ): """Store extracted elements in database with optimized bulk insert""" BATCH_SIZE = 5000 # Large batch size for fast bulk insert total_elements = len(elements_data) logger.info(f"Storing {total_elements} elements in batches of {BATCH_SIZE}") # Delete existing elements for this project before inserting new ones from sqlalchemy import delete logger.info(f"Deleting existing elements for project {project.id}") await session.execute(delete(Element).where(Element.project_id == project.id)) await session.commit() logger.info(f"Deleted existing elements for project {project.id}") # Prepare elements in batches for bulk insert for batch_start in range(0, total_elements, BATCH_SIZE): batch_end = min(batch_start + BATCH_SIZE, total_elements) batch_data = elements_data[batch_start:batch_end] # Create element objects for this batch elements_batch = [] for element_data in batch_data: try: element = Element( project_id=project.id, ifc_global_id=element_data.get("ifc_global_id"), ifc_type=element_data.get("ifc_type"), element_type=element_data.get("element_type"), ifc_class=element_data.get("ifc_class"), name=element_data.get("name"), description=element_data.get("description"), tag=element_data.get("tag"), level_name=element_data.get("level_name"), level_elevation=element_data.get("level_elevation"), building_name=element_data.get("building_name"), site_name=element_data.get("site_name"), bounding_box=element_data.get("bounding_box"), centroid=element_data.get("centroid"), area=element_data.get("area"), volume=element_data.get("volume"), length=element_data.get("length"), width=element_data.get("width"), height=element_data.get("height"), perimeter=element_data.get("perimeter"), material_name=element_data.get("material_name"), material_category=element_data.get("material_category"), material_properties=element_data.get("material_properties"), properties=element_data.get("properties", {}), quantities=element_data.get("quantities", {}), classifications=element_data.get("classifications", {}), parent_element_id=element_data.get("parent_element_id"), assembly_id=element_data.get("assembly_id"), has_geometry=element_data.get("has_geometry", False), has_properties=bool(element_data.get("properties")), has_materials=bool(element_data.get("material_name")), is_external=element_data.get("is_external"), is_load_bearing=element_data.get("is_load_bearing"), fire_rating=element_data.get("fire_rating"), combustible=element_data.get("combustible"), acoustic_rating=element_data.get("acoustic_rating"), thermal_transmittance=element_data.get("thermal_transmittance"), thermal_mass=element_data.get("thermal_mass") ) elements_batch.append(element) except Exception as e: logger.warning(f"Failed to prepare element {element_data.get('ifc_global_id')}: {e}") continue # Bulk insert the batch if elements_batch: try: session.add_all(elements_batch) await session.commit() logger.info(f"Committed batch: {batch_end}/{total_elements} elements") # Update progress every 5 batches to reduce DB commits batch_index = batch_start // BATCH_SIZE if task and project_id and (batch_index % 5 == 0 or batch_end >= total_elements): progress_percent = 60 + int(batch_end / total_elements * 20) # 60-80% task.update_state( state="PROGRESS", meta={"stage": "storing_elements", "percent": progress_percent} ) _update_redis_progress(project_id, progress_percent, "storing_elements") except Exception as e: await session.rollback() logger.error(f"Failed to commit batch at {batch_end}: {e}") raise logger.info(f"Stored all {total_elements} elements in database") async def _mark_project_failed(project_id: str, error_message: str): """Mark project as failed""" try: async_session_maker = get_async_session_maker() async with async_session_maker() as session: internal_id = int(project_id.replace('prj_', '')) query = select(Project).where(Project.id == internal_id) result = await session.execute(query) project = result.scalar_one_or_none() if project: project.mark_failed(error_message) await session.commit() logger.info(f"Marked project {project_id} as failed") except Exception as e: logger.error(f"Failed to mark project {project_id} as failed: {e}") @celery_app.task( bind=True, max_retries=1, acks_late=True, time_limit=7200, soft_time_limit=6000 ) def enrich_geometry(self, project_data: Dict[str, Any]): """ Background geometry enrichment — extracts bounding boxes and dimensions from IFC geometry for elements that don't have them from properties. Runs AFTER the fast import completes and project is already 'ready'. """ project_id = project_data.get("project_id") s3_key = project_data.get("s3_key") filename = project_data.get("filename") logger.info(f"Starting geometry enrichment for project {project_id}") try: asyncio.run(_enrich_geometry_async(project_data)) logger.info(f"Geometry enrichment completed for project {project_id}") return {"status": "completed", "project_id": project_id} except RuntimeError as e: if "running event loop" in str(e): import concurrent.futures def run_in_thread(): new_loop = asyncio.new_event_loop() asyncio.set_event_loop(new_loop) try: return new_loop.run_until_complete(_enrich_geometry_async(project_data)) finally: new_loop.close() with concurrent.futures.ThreadPoolExecutor() as executor: executor.submit(run_in_thread).result() return {"status": "completed", "project_id": project_id} raise except Exception as e: logger.error(f"Geometry enrichment failed for project {project_id}: {e}") return {"status": "failed", "project_id": project_id, "error": str(e)} async def _enrich_geometry_async(project_data: Dict[str, Any]): """Async geometry enrichment — downloads IFC, extracts geometry, updates DB""" project_id = project_data.get("project_id") s3_key = project_data.get("s3_key") filename = project_data.get("filename") internal_id = int(project_id.replace('prj_', '')) async_session_maker = get_async_session_maker() async with async_session_maker() as session: # Download IFC file from S3 from app.core.s3 import s3_client, init_s3, S3StorageService if not s3_client: init_s3() file_content = S3StorageService.download_file_by_key(s3_key) # Open IFC model import tempfile import ifcopenshell import ifcopenshell.geom import numpy as np import stat fd = tempfile.mkstemp(suffix='.ifc') temp_path = fd[1] os.close(fd[0]) os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR) # 0o600 with open(temp_path, 'wb') as temp_file: temp_file.write(file_content) try: ifc_file = ifcopenshell.open(temp_path) # Configure geometry settings geom_settings = ifcopenshell.geom.settings() geom_settings.set(geom_settings.USE_WORLD_COORDS, True) from app.services.gpu_accelerator import get_gpu_accelerator gpu = get_gpu_accelerator() products = ifc_file.by_type('IfcProduct') logger.info(f"Enriching geometry for {len(products)} products in project {project_id}") # Load all elements from DB for this project (id → element mapping) from app.models import Element result = await session.execute( select(Element).where(Element.project_id == internal_id) ) elements = {elem.ifc_global_id: elem for elem in result.scalars().all()} updated_count = 0 import concurrent.futures import os # Use ifcopenshell's native C++ multi-threaded iterator for maximum speed num_threads = os.cpu_count() or 6 logger.info(f"Geometry enrichment using native iterator with {num_threads} C++ threads") it = ifcopenshell.geom.iterator(geom_settings, ifc_file, num_threads=num_threads) total_processed = 0 if it.initialize(): while True: shape = it.get() total_processed += 1 try: global_id = shape.guid verts = shape.geometry.verts vertices = np.array(verts).reshape(-1, 3) if len(vertices) > 0: bbox = gpu.calculate_bounding_box_batch([vertices])[0] centroid = gpu.calculate_centroids_batch([vertices])[0] min_pt, max_pt = bbox["min"], bbox["max"] width = abs(max_pt[0] - min_pt[0]) depth = abs(max_pt[1] - min_pt[1]) height_val = abs(max_pt[2] - min_pt[2]) elem = elements.get(global_id) if elem: elem.bounding_box = bbox elem.centroid = centroid elem.has_geometry = True ifc_type = elem.ifc_type or "" # Calculate area based on element type area = None if ifc_type in ['IfcWindow', 'IfcDoor']: area = width * height_val elif ifc_type in ['IfcSlab', 'IfcRoof', 'IfcCovering']: area = width * depth elif ifc_type in ['IfcWall', 'IfcCurtainWall']: area = (width + depth) * 2 * height_val # Update only if not already set from IFC properties if elem.area is None and area and area > 0.001: elem.area = round(area, 3) volume = width * depth * height_val if elem.volume is None and volume > 0.001: elem.volume = round(volume, 3) if elem.height is None and height_val > 0.001: elem.height = round(height_val, 3) if elem.width is None and width > 0.001: elem.width = round(width, 3) if elem.length is None and ifc_type in ['IfcBeam', 'IfcColumn', 'IfcMember']: elem.length = round(max(width, depth, height_val), 3) # Perimeter for planar elements if elem.perimeter is None: if ifc_type in ['IfcWall', 'IfcCurtainWall', 'IfcSlab', 'IfcRoof', 'IfcCovering']: perimeter = (width + depth) * 2 if perimeter > 0.001: elem.perimeter = round(perimeter, 3) elif ifc_type in ['IfcWindow', 'IfcDoor']: perimeter = (width + height_val) * 2 if perimeter > 0.001: elem.perimeter = round(perimeter, 3) updated_count += 1 except Exception as e: logger.debug(f"Geometry extraction failed for element: {e}") # Commit and log every 500 elements if total_processed % 500 == 0: await session.commit() logger.info(f"Geometry enrichment: {total_processed} processed, {updated_count} updated") if not it.next(): break await session.commit() # Update project flags project = await session.get(Project, internal_id) if project: project.has_geometry = updated_count > 0 await session.commit() logger.info(f"Geometry enrichment done: {updated_count}/{len(products)} elements updated") finally: try: os.unlink(temp_path) except OSError: pass @celery_app.task(bind=True) def reprocess_failed_project(self, project_id: str): """ Retry processing for a failed project """ logger.info(f"Retrying processing for failed project {project_id}") # This would fetch the original project data and retry processing # Implementation would depend on how we store the original upload data return {"status": "retry_initiated", "project_id": project_id} @celery_app.task def cleanup_processing_artifacts(project_id: str): """ Clean up temporary files and artifacts from processing """ logger.info(f"Cleaning up processing artifacts for project {project_id}") try: # Clean up any temporary files, cache entries, etc. # This could include: # - Temporary IFC processing files # - Cached model data # - Intermediate processing results return {"status": "cleaned", "project_id": project_id} except Exception as e: logger.error(f"Failed to clean up artifacts for project {project_id}: {e}") return {"status": "cleanup_failed", "project_id": project_id, "error": str(e)} @celery_app.task def process_ifc_stream(): """ Process IFC events from Redis Stream """ import threading import concurrent.futures # Check if we're in an event loop already try: loop = asyncio.get_running_loop() # If we're in a loop, use a new thread def run_in_thread(): new_loop = asyncio.new_event_loop() asyncio.set_event_loop(new_loop) try: return new_loop.run_until_complete(_process_ifc_stream_async()) finally: new_loop.close() with concurrent.futures.ThreadPoolExecutor() as executor: future = executor.submit(run_in_thread) return future.result() except RuntimeError: # No running loop, use asyncio.run directly return asyncio.run(_process_ifc_stream_async()) async def _process_ifc_stream_async(): """Process IFC events from Redis Stream""" processed_count = 0 try: logger.info("Processing IFC events from Redis Stream") # Each Celery task gets a fresh asyncio event loop (via # asyncio.run / new_event_loop), but the global redis_client # outlives the previous loop and is therefore tied to a closed # one — calls would raise "Event loop is closed". Re-init each # invocation; the orphaned async client gets garbage-collected. from app.core.redis import init_redis await init_redis() # Read from IFC processing stream messages = await RedisStream.read_events( stream="ifc.process", consumer_group="ifc_processors", consumer_name="worker-1", count=10, block=1000 ) for stream, msgs in messages: for msg_id, fields in msgs: try: # Extract project data from stream event project_data = {} for i in range(0, len(fields), 2): key = fields[i].decode() if isinstance(fields[i], bytes) else fields[i] value = fields[i + 1].decode() if isinstance(fields[i + 1], bytes) else fields[i + 1] project_data[key] = value project_id = project_data.get("project_id") logger.info(f"Processing IFC stream event for project {project_id}") # Trigger the actual processing task process_ifc_file.delay(project_data) processed_count += 1 # Acknowledge the message await RedisStream.ack_event("ifc.process", "ifc_processors", msg_id) except Exception as e: logger.error(f"Error processing IFC stream message {msg_id}: {e}") continue logger.info(f"Processed {processed_count} IFC events from stream") return {"processed": processed_count} except Exception as e: logger.error(f"Error processing IFC stream: {e}") return {"error": str(e)}