Spaces:
Runtime error
Runtime error
BIM Query Deploy
Claude Opus 4.7 (1M context)
deploy: pick up R2/health/alembic/celery fixes from monorepo
0917996 Download app/tasks/ifc_processing.py from 12mhsssss/bimquery-api: direct link, hf CLI and curl.
- Browser
- Download file 34.5 kB
-
https://huggingface.co/spaces/12mhsssss/bimquery-api/resolve/main/app/tasks/ifc_processing.py
- Command line
-
hf download hf://spaces/12mhsssss/bimquery-api/app/tasks/ifc_processing.py
-
curl -L -o ifc_processing.py https://huggingface.co/spaces/12mhsssss/bimquery-api/resolve/main/app/tasks/ifc_processing.py
34.5 kB
| # Copyright (C) 2026 Mattia Zatelli | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU Affero General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU Affero General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU Affero General Public License | |
| # along with this program. If not, see <https://www.gnu.org/licenses/>. | |
| """ | |
| IFC processing Celery tasks | |
| """ | |
| import asyncio | |
| import json as _json | |
| import logging | |
| import os | |
| import traceback | |
| from datetime import datetime | |
| from typing import Dict, Any | |
| from celery import current_task | |
| from sqlalchemy import select | |
| from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession | |
| from app.core.celery_app import celery_app | |
| from app.core.config import get_settings | |
| from app.core.s3 import S3StorageService | |
| from app.core.redis import RedisStream | |
| from app.models import Project, ProjectStatus, Element | |
| from app.services.ifc_processor import IFCProcessor | |
| logger = logging.getLogger(__name__) | |
| settings = get_settings() | |
| # Don't create engine globally - create per task to avoid event loop issues | |
| def get_async_session_maker(): | |
| """Create async session maker per task to avoid event loop conflicts""" | |
| engine = create_async_engine( | |
| settings.DATABASE_URL, | |
| pool_pre_ping=True, | |
| pool_recycle=3600 | |
| ) | |
| return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) | |
| def process_ifc_file(self, project_data: Dict[str, Any]): | |
| """ | |
| Process uploaded IFC file asynchronously | |
| Args: | |
| project_data: Dictionary containing project information from upload | |
| """ | |
| # Validate required task parameters | |
| required_fields = ("project_id", "tenant_id", "s3_key", "filename") | |
| missing = [f for f in required_fields if not project_data.get(f)] | |
| if missing: | |
| logger.error(f"Rejecting task: missing required fields: {missing}") | |
| return {"status": "rejected", "error": f"Missing fields: {missing}"} | |
| project_id = project_data["project_id"] | |
| tenant_id = project_data["tenant_id"] | |
| s3_bucket = project_data.get("s3_bucket") | |
| s3_key = project_data["s3_key"] | |
| filename = project_data["filename"] | |
| # Validate project_id format | |
| if not project_id.startswith("prj_"): | |
| logger.error(f"Rejecting task: invalid project_id format: {project_id}") | |
| return {"status": "rejected", "error": "Invalid project_id format"} | |
| # Validate S3 key prefix to prevent path traversal | |
| if not s3_key.startswith("projects/") or ".." in s3_key: | |
| logger.error(f"Rejecting task: invalid s3_key: {s3_key}") | |
| return {"status": "rejected", "error": "Invalid s3_key"} | |
| logger.info(f"Starting IFC processing for project {project_id}") | |
| # Implement Redis lock to prevent duplicate task execution | |
| from app.core.redis import redis_client | |
| lock_key = f"ifc_processing_lock:{project_id}" | |
| # Try to acquire lock with 30 minute timeout | |
| lock = redis_client.lock(lock_key, timeout=1800, blocking=False) | |
| if not lock.acquire(blocking=False): | |
| logger.warning(f"Another task is already processing project {project_id}, skipping duplicate") | |
| return { | |
| "status": "skipped", | |
| "project_id": project_id, | |
| "message": "Another processing task is already running for this project" | |
| } | |
| try: | |
| # Update task progress | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "initializing", "percent": 0} | |
| ) | |
| # Run the async processing | |
| # In Celery worker, we should NOT have a running loop, so use asyncio.run | |
| try: | |
| asyncio.run(_process_ifc_async(project_data)) | |
| except RuntimeError as e: | |
| # If there's already a loop running, create a new one in a thread | |
| if "running event loop" in str(e): | |
| logger.warning("Detected running event loop, creating new loop in thread") | |
| import concurrent.futures | |
| def run_in_thread(): | |
| new_loop = asyncio.new_event_loop() | |
| asyncio.set_event_loop(new_loop) | |
| try: | |
| return new_loop.run_until_complete(_process_ifc_async(project_data)) | |
| finally: | |
| new_loop.close() | |
| asyncio.set_event_loop(None) | |
| with concurrent.futures.ThreadPoolExecutor() as executor: | |
| future = executor.submit(run_in_thread) | |
| future.result() | |
| else: | |
| raise | |
| logger.info(f"IFC processing completed for project {project_id}") | |
| return { | |
| "status": "completed", | |
| "project_id": project_id, | |
| "message": "IFC file processed successfully" | |
| } | |
| except Exception as exc: | |
| logger.error(f"IFC processing failed for project {project_id}: {exc}") | |
| logger.error(traceback.format_exc()) | |
| # Update project status to failed | |
| try: | |
| asyncio.run(_mark_project_failed(project_id, str(exc))) | |
| except RuntimeError as e: | |
| if "running event loop" in str(e): | |
| logger.warning("Detected running event loop when marking failed") | |
| import concurrent.futures | |
| def mark_failed_in_thread(): | |
| new_loop = asyncio.new_event_loop() | |
| asyncio.set_event_loop(new_loop) | |
| try: | |
| return new_loop.run_until_complete(_mark_project_failed(project_id, str(exc))) | |
| finally: | |
| new_loop.close() | |
| asyncio.set_event_loop(None) | |
| with concurrent.futures.ThreadPoolExecutor() as executor: | |
| future = executor.submit(mark_failed_in_thread) | |
| future.result() | |
| else: | |
| logger.error(f"Failed to mark project as failed: {e}") | |
| # Retry logic | |
| if self.request.retries < self.max_retries: | |
| logger.info(f"Retrying IFC processing for project {project_id} (attempt {self.request.retries + 1})") | |
| raise self.retry(exc=exc, countdown=60 * (self.request.retries + 1)) | |
| # Final failure | |
| return { | |
| "status": "failed", | |
| "project_id": project_id, | |
| "error": str(exc) | |
| } | |
| finally: | |
| # Always release the lock | |
| try: | |
| lock.release() | |
| logger.info(f"Released processing lock for project {project_id}") | |
| except Exception as e: | |
| logger.warning(f"Failed to release lock for project {project_id}: {e}") | |
| def _update_redis_progress(project_id: str, percent: int, stage: str): | |
| """Write real-time progress to Redis for the /status endpoint""" | |
| try: | |
| from app.core.redis import redis_client | |
| if redis_client: | |
| redis_client.set( | |
| f"project_progress:{project_id}", | |
| _json.dumps({"percent": percent, "stage": stage}), | |
| ex=600 | |
| ) | |
| except Exception: | |
| pass | |
| async def _process_ifc_async(project_data: Dict[str, Any]): | |
| """Async IFC processing logic""" | |
| project_id = project_data.get("project_id") | |
| s3_bucket = project_data.get("s3_bucket") | |
| s3_key = project_data.get("s3_key") | |
| filename = project_data.get("filename") | |
| logger.info(f"DEBUG: Starting async processing for project {project_id} with s3_key: {s3_key}") | |
| # Create session maker for this task instance | |
| async_session_maker = get_async_session_maker() | |
| async with async_session_maker() as session: | |
| # Get project record | |
| try: | |
| internal_id = int(project_id.replace('prj_', '')) | |
| except ValueError: | |
| raise ValueError(f"Invalid project ID format: {project_id}") | |
| query = select(Project).where(Project.id == internal_id) | |
| result = await session.execute(query) | |
| project = result.scalar_one_or_none() | |
| if not project: | |
| raise ValueError(f"Project not found: {project_id}") | |
| try: | |
| # Mark as processing | |
| project.mark_processing("downloading") | |
| await session.commit() | |
| # Update progress | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "downloading", "percent": 10} | |
| ) | |
| # Initialize S3 client if not already done | |
| from app.core.s3 import s3_client, init_s3 | |
| if not s3_client: | |
| logger.info("Initializing S3 client for IFC processing") | |
| init_s3() | |
| # Download file from S3 | |
| logger.info(f"DEBUG: Starting download for project {project_id} from S3 key: {s3_key}") | |
| file_content = S3StorageService.download_file_by_key(s3_key) | |
| logger.info(f"DEBUG: Download completed, file size: {len(file_content)} bytes") | |
| # Update progress | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "parsing", "percent": 20} | |
| ) | |
| # Initialize IFC processor | |
| processor = IFCProcessor() | |
| # Parse IFC file | |
| logger.info(f"DEBUG: Starting IFC parsing for project {project_id}") | |
| ifc_data = await processor.parse_ifc_content(file_content, filename) | |
| logger.info(f"DEBUG: IFC parsing completed, found {len(ifc_data.get('elements', []))} elements") | |
| # Update project metadata | |
| project.ifc_schema = ifc_data.get("schema") | |
| project.ifc_version = ifc_data.get("version") | |
| # Store project info with units | |
| project_info = ifc_data.get("project_info", {}) | |
| units = ifc_data.get("units", {}) | |
| logger.info(f"DEBUG: Extracted units for project {project_id}: {units}") | |
| project_info['units'] = units | |
| project.project_info = project_info | |
| # Force SQLAlchemy to detect the change in JSONB field | |
| from sqlalchemy.orm.attributes import flag_modified | |
| flag_modified(project, "project_info") | |
| logger.info(f"DEBUG: Set project_info with units for project {project_id}: {project.project_info}") | |
| project.site_info = ifc_data.get("site_info", {}) | |
| project.bounding_box = ifc_data.get("bounding_box") | |
| project.gross_floor_area = ifc_data.get("gross_floor_area") | |
| project.building_height = ifc_data.get("building_height") | |
| project.update_progress(40, "extracting_elements") | |
| await session.commit() | |
| logger.info(f"DEBUG: Committed project metadata for project {project_id}") | |
| # Update progress | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "extracting_elements", "percent": 40} | |
| ) | |
| # Extract elements with progress callback | |
| logger.info(f"Extracting elements for project {project_id}") | |
| def extraction_progress(current, total): | |
| """Callback for extraction progress updates (sync, called from thread pool)""" | |
| try: | |
| progress_percent = 40 + int(current / total * 20) | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "extracting_elements", "percent": progress_percent} | |
| ) | |
| _update_redis_progress(project_id, progress_percent, "extracting_elements") | |
| except Exception as e: | |
| logger.warning(f"Error in extraction progress callback: {e}") | |
| elements_data = await processor.extract_elements(ifc_data.get("ifc_model"), extraction_progress) | |
| # Persist final extraction progress to DB | |
| project.update_progress(60, "extraction_complete") | |
| await session.commit() | |
| logger.info(f"Extraction completed: {len(elements_data)} elements") | |
| # Update progress | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "storing_elements", "percent": 60} | |
| ) | |
| _update_redis_progress(project_id, 60, "storing_elements") | |
| # Store elements in database | |
| logger.info(f"DEBUG: Starting database storage of {len(elements_data)} elements for project {project_id}") | |
| await _store_elements(session, project, elements_data, current_task, project_id) | |
| logger.info(f"DEBUG: Database storage completed for project {project_id}") | |
| project.update_progress(80, "indexing") | |
| await session.commit() | |
| # Update progress | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "indexing", "percent": 80} | |
| ) | |
| # Create search indexes and embeddings | |
| await processor.create_search_indexes(project.id, elements_data) | |
| project.update_progress(95, "finalizing") | |
| current_task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "finalizing", "percent": 95} | |
| ) | |
| _update_redis_progress(project_id, 95, "finalizing") | |
| # Calculate final statistics in single pass | |
| element_count = len(elements_data) | |
| element_types = {} | |
| has_geometry = False | |
| has_properties = False | |
| has_materials = False | |
| has_spaces = False | |
| for elem in elements_data: | |
| ifc_type = elem.get("ifc_type", "Unknown") | |
| element_types[ifc_type] = element_types.get(ifc_type, 0) + 1 | |
| if not has_geometry and elem.get("has_geometry"): | |
| has_geometry = True | |
| if not has_properties and elem.get("properties"): | |
| has_properties = True | |
| if not has_materials and elem.get("material_name"): | |
| has_materials = True | |
| if not has_spaces and ifc_type == "IfcSpace": | |
| has_spaces = True | |
| # Update project final status | |
| project.element_count = element_count | |
| project.element_types = element_types | |
| project.has_geometry = has_geometry | |
| project.has_properties = has_properties | |
| project.has_materials = has_materials | |
| project.has_spaces = has_spaces | |
| project.mark_ready() | |
| await session.commit() | |
| # Final update | |
| current_task.update_state( | |
| state="SUCCESS", | |
| meta={"stage": "ready", "percent": 100} | |
| ) | |
| _update_redis_progress(project_id, 100, "ready") | |
| logger.info(f"IFC processing completed for project {project_id}: {element_count} elements") | |
| # Schedule background geometry enrichment (non-blocking) | |
| try: | |
| enrich_geometry.delay({ | |
| "project_id": project_id, | |
| "internal_id": internal_id, | |
| "s3_key": s3_key, | |
| "filename": filename, | |
| }) | |
| logger.info(f"Scheduled geometry enrichment for project {project_id}") | |
| except Exception as e: | |
| logger.warning(f"Could not schedule geometry enrichment: {e}") | |
| except Exception as e: | |
| project.mark_failed(str(e), {"traceback": traceback.format_exc()}) | |
| await session.commit() | |
| raise | |
| async def _store_elements( | |
| session: AsyncSession, | |
| project: Project, | |
| elements_data: list, | |
| task=None, | |
| project_id: str = None | |
| ): | |
| """Store extracted elements in database with optimized bulk insert""" | |
| BATCH_SIZE = 5000 # Large batch size for fast bulk insert | |
| total_elements = len(elements_data) | |
| logger.info(f"Storing {total_elements} elements in batches of {BATCH_SIZE}") | |
| # Delete existing elements for this project before inserting new ones | |
| from sqlalchemy import delete | |
| logger.info(f"Deleting existing elements for project {project.id}") | |
| await session.execute(delete(Element).where(Element.project_id == project.id)) | |
| await session.commit() | |
| logger.info(f"Deleted existing elements for project {project.id}") | |
| # Prepare elements in batches for bulk insert | |
| for batch_start in range(0, total_elements, BATCH_SIZE): | |
| batch_end = min(batch_start + BATCH_SIZE, total_elements) | |
| batch_data = elements_data[batch_start:batch_end] | |
| # Create element objects for this batch | |
| elements_batch = [] | |
| for element_data in batch_data: | |
| try: | |
| element = Element( | |
| project_id=project.id, | |
| ifc_global_id=element_data.get("ifc_global_id"), | |
| ifc_type=element_data.get("ifc_type"), | |
| element_type=element_data.get("element_type"), | |
| ifc_class=element_data.get("ifc_class"), | |
| name=element_data.get("name"), | |
| description=element_data.get("description"), | |
| tag=element_data.get("tag"), | |
| level_name=element_data.get("level_name"), | |
| level_elevation=element_data.get("level_elevation"), | |
| building_name=element_data.get("building_name"), | |
| site_name=element_data.get("site_name"), | |
| bounding_box=element_data.get("bounding_box"), | |
| centroid=element_data.get("centroid"), | |
| area=element_data.get("area"), | |
| volume=element_data.get("volume"), | |
| length=element_data.get("length"), | |
| width=element_data.get("width"), | |
| height=element_data.get("height"), | |
| perimeter=element_data.get("perimeter"), | |
| material_name=element_data.get("material_name"), | |
| material_category=element_data.get("material_category"), | |
| material_properties=element_data.get("material_properties"), | |
| properties=element_data.get("properties", {}), | |
| quantities=element_data.get("quantities", {}), | |
| classifications=element_data.get("classifications", {}), | |
| parent_element_id=element_data.get("parent_element_id"), | |
| assembly_id=element_data.get("assembly_id"), | |
| has_geometry=element_data.get("has_geometry", False), | |
| has_properties=bool(element_data.get("properties")), | |
| has_materials=bool(element_data.get("material_name")), | |
| is_external=element_data.get("is_external"), | |
| is_load_bearing=element_data.get("is_load_bearing"), | |
| fire_rating=element_data.get("fire_rating"), | |
| combustible=element_data.get("combustible"), | |
| acoustic_rating=element_data.get("acoustic_rating"), | |
| thermal_transmittance=element_data.get("thermal_transmittance"), | |
| thermal_mass=element_data.get("thermal_mass") | |
| ) | |
| elements_batch.append(element) | |
| except Exception as e: | |
| logger.warning(f"Failed to prepare element {element_data.get('ifc_global_id')}: {e}") | |
| continue | |
| # Bulk insert the batch | |
| if elements_batch: | |
| try: | |
| session.add_all(elements_batch) | |
| await session.commit() | |
| logger.info(f"Committed batch: {batch_end}/{total_elements} elements") | |
| # Update progress every 5 batches to reduce DB commits | |
| batch_index = batch_start // BATCH_SIZE | |
| if task and project_id and (batch_index % 5 == 0 or batch_end >= total_elements): | |
| progress_percent = 60 + int(batch_end / total_elements * 20) # 60-80% | |
| task.update_state( | |
| state="PROGRESS", | |
| meta={"stage": "storing_elements", "percent": progress_percent} | |
| ) | |
| _update_redis_progress(project_id, progress_percent, "storing_elements") | |
| except Exception as e: | |
| await session.rollback() | |
| logger.error(f"Failed to commit batch at {batch_end}: {e}") | |
| raise | |
| logger.info(f"Stored all {total_elements} elements in database") | |
| async def _mark_project_failed(project_id: str, error_message: str): | |
| """Mark project as failed""" | |
| try: | |
| async_session_maker = get_async_session_maker() | |
| async with async_session_maker() as session: | |
| internal_id = int(project_id.replace('prj_', '')) | |
| query = select(Project).where(Project.id == internal_id) | |
| result = await session.execute(query) | |
| project = result.scalar_one_or_none() | |
| if project: | |
| project.mark_failed(error_message) | |
| await session.commit() | |
| logger.info(f"Marked project {project_id} as failed") | |
| except Exception as e: | |
| logger.error(f"Failed to mark project {project_id} as failed: {e}") | |
| def enrich_geometry(self, project_data: Dict[str, Any]): | |
| """ | |
| Background geometry enrichment — extracts bounding boxes and dimensions | |
| from IFC geometry for elements that don't have them from properties. | |
| Runs AFTER the fast import completes and project is already 'ready'. | |
| """ | |
| project_id = project_data.get("project_id") | |
| s3_key = project_data.get("s3_key") | |
| filename = project_data.get("filename") | |
| logger.info(f"Starting geometry enrichment for project {project_id}") | |
| try: | |
| asyncio.run(_enrich_geometry_async(project_data)) | |
| logger.info(f"Geometry enrichment completed for project {project_id}") | |
| return {"status": "completed", "project_id": project_id} | |
| except RuntimeError as e: | |
| if "running event loop" in str(e): | |
| import concurrent.futures | |
| def run_in_thread(): | |
| new_loop = asyncio.new_event_loop() | |
| asyncio.set_event_loop(new_loop) | |
| try: | |
| return new_loop.run_until_complete(_enrich_geometry_async(project_data)) | |
| finally: | |
| new_loop.close() | |
| with concurrent.futures.ThreadPoolExecutor() as executor: | |
| executor.submit(run_in_thread).result() | |
| return {"status": "completed", "project_id": project_id} | |
| raise | |
| except Exception as e: | |
| logger.error(f"Geometry enrichment failed for project {project_id}: {e}") | |
| return {"status": "failed", "project_id": project_id, "error": str(e)} | |
| async def _enrich_geometry_async(project_data: Dict[str, Any]): | |
| """Async geometry enrichment — downloads IFC, extracts geometry, updates DB""" | |
| project_id = project_data.get("project_id") | |
| s3_key = project_data.get("s3_key") | |
| filename = project_data.get("filename") | |
| internal_id = int(project_id.replace('prj_', '')) | |
| async_session_maker = get_async_session_maker() | |
| async with async_session_maker() as session: | |
| # Download IFC file from S3 | |
| from app.core.s3 import s3_client, init_s3, S3StorageService | |
| if not s3_client: | |
| init_s3() | |
| file_content = S3StorageService.download_file_by_key(s3_key) | |
| # Open IFC model | |
| import tempfile | |
| import ifcopenshell | |
| import ifcopenshell.geom | |
| import numpy as np | |
| import stat | |
| fd = tempfile.mkstemp(suffix='.ifc') | |
| temp_path = fd[1] | |
| os.close(fd[0]) | |
| os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR) # 0o600 | |
| with open(temp_path, 'wb') as temp_file: | |
| temp_file.write(file_content) | |
| try: | |
| ifc_file = ifcopenshell.open(temp_path) | |
| # Configure geometry settings | |
| geom_settings = ifcopenshell.geom.settings() | |
| geom_settings.set(geom_settings.USE_WORLD_COORDS, True) | |
| from app.services.gpu_accelerator import get_gpu_accelerator | |
| gpu = get_gpu_accelerator() | |
| products = ifc_file.by_type('IfcProduct') | |
| logger.info(f"Enriching geometry for {len(products)} products in project {project_id}") | |
| # Load all elements from DB for this project (id → element mapping) | |
| from app.models import Element | |
| result = await session.execute( | |
| select(Element).where(Element.project_id == internal_id) | |
| ) | |
| elements = {elem.ifc_global_id: elem for elem in result.scalars().all()} | |
| updated_count = 0 | |
| import concurrent.futures | |
| import os | |
| # Use ifcopenshell's native C++ multi-threaded iterator for maximum speed | |
| num_threads = os.cpu_count() or 6 | |
| logger.info(f"Geometry enrichment using native iterator with {num_threads} C++ threads") | |
| it = ifcopenshell.geom.iterator(geom_settings, ifc_file, num_threads=num_threads) | |
| total_processed = 0 | |
| if it.initialize(): | |
| while True: | |
| shape = it.get() | |
| total_processed += 1 | |
| try: | |
| global_id = shape.guid | |
| verts = shape.geometry.verts | |
| vertices = np.array(verts).reshape(-1, 3) | |
| if len(vertices) > 0: | |
| bbox = gpu.calculate_bounding_box_batch([vertices])[0] | |
| centroid = gpu.calculate_centroids_batch([vertices])[0] | |
| min_pt, max_pt = bbox["min"], bbox["max"] | |
| width = abs(max_pt[0] - min_pt[0]) | |
| depth = abs(max_pt[1] - min_pt[1]) | |
| height_val = abs(max_pt[2] - min_pt[2]) | |
| elem = elements.get(global_id) | |
| if elem: | |
| elem.bounding_box = bbox | |
| elem.centroid = centroid | |
| elem.has_geometry = True | |
| ifc_type = elem.ifc_type or "" | |
| # Calculate area based on element type | |
| area = None | |
| if ifc_type in ['IfcWindow', 'IfcDoor']: | |
| area = width * height_val | |
| elif ifc_type in ['IfcSlab', 'IfcRoof', 'IfcCovering']: | |
| area = width * depth | |
| elif ifc_type in ['IfcWall', 'IfcCurtainWall']: | |
| area = (width + depth) * 2 * height_val | |
| # Update only if not already set from IFC properties | |
| if elem.area is None and area and area > 0.001: | |
| elem.area = round(area, 3) | |
| volume = width * depth * height_val | |
| if elem.volume is None and volume > 0.001: | |
| elem.volume = round(volume, 3) | |
| if elem.height is None and height_val > 0.001: | |
| elem.height = round(height_val, 3) | |
| if elem.width is None and width > 0.001: | |
| elem.width = round(width, 3) | |
| if elem.length is None and ifc_type in ['IfcBeam', 'IfcColumn', 'IfcMember']: | |
| elem.length = round(max(width, depth, height_val), 3) | |
| # Perimeter for planar elements | |
| if elem.perimeter is None: | |
| if ifc_type in ['IfcWall', 'IfcCurtainWall', 'IfcSlab', 'IfcRoof', 'IfcCovering']: | |
| perimeter = (width + depth) * 2 | |
| if perimeter > 0.001: | |
| elem.perimeter = round(perimeter, 3) | |
| elif ifc_type in ['IfcWindow', 'IfcDoor']: | |
| perimeter = (width + height_val) * 2 | |
| if perimeter > 0.001: | |
| elem.perimeter = round(perimeter, 3) | |
| updated_count += 1 | |
| except Exception as e: | |
| logger.debug(f"Geometry extraction failed for element: {e}") | |
| # Commit and log every 500 elements | |
| if total_processed % 500 == 0: | |
| await session.commit() | |
| logger.info(f"Geometry enrichment: {total_processed} processed, {updated_count} updated") | |
| if not it.next(): | |
| break | |
| await session.commit() | |
| # Update project flags | |
| project = await session.get(Project, internal_id) | |
| if project: | |
| project.has_geometry = updated_count > 0 | |
| await session.commit() | |
| logger.info(f"Geometry enrichment done: {updated_count}/{len(products)} elements updated") | |
| finally: | |
| try: | |
| os.unlink(temp_path) | |
| except OSError: | |
| pass | |
| def reprocess_failed_project(self, project_id: str): | |
| """ | |
| Retry processing for a failed project | |
| """ | |
| logger.info(f"Retrying processing for failed project {project_id}") | |
| # This would fetch the original project data and retry processing | |
| # Implementation would depend on how we store the original upload data | |
| return {"status": "retry_initiated", "project_id": project_id} | |
| def cleanup_processing_artifacts(project_id: str): | |
| """ | |
| Clean up temporary files and artifacts from processing | |
| """ | |
| logger.info(f"Cleaning up processing artifacts for project {project_id}") | |
| try: | |
| # Clean up any temporary files, cache entries, etc. | |
| # This could include: | |
| # - Temporary IFC processing files | |
| # - Cached model data | |
| # - Intermediate processing results | |
| return {"status": "cleaned", "project_id": project_id} | |
| except Exception as e: | |
| logger.error(f"Failed to clean up artifacts for project {project_id}: {e}") | |
| return {"status": "cleanup_failed", "project_id": project_id, "error": str(e)} | |
| def process_ifc_stream(): | |
| """ | |
| Process IFC events from Redis Stream | |
| """ | |
| import threading | |
| import concurrent.futures | |
| # Check if we're in an event loop already | |
| try: | |
| loop = asyncio.get_running_loop() | |
| # If we're in a loop, use a new thread | |
| def run_in_thread(): | |
| new_loop = asyncio.new_event_loop() | |
| asyncio.set_event_loop(new_loop) | |
| try: | |
| return new_loop.run_until_complete(_process_ifc_stream_async()) | |
| finally: | |
| new_loop.close() | |
| with concurrent.futures.ThreadPoolExecutor() as executor: | |
| future = executor.submit(run_in_thread) | |
| return future.result() | |
| except RuntimeError: | |
| # No running loop, use asyncio.run directly | |
| return asyncio.run(_process_ifc_stream_async()) | |
| async def _process_ifc_stream_async(): | |
| """Process IFC events from Redis Stream""" | |
| processed_count = 0 | |
| try: | |
| logger.info("Processing IFC events from Redis Stream") | |
| # Each Celery task gets a fresh asyncio event loop (via | |
| # asyncio.run / new_event_loop), but the global redis_client | |
| # outlives the previous loop and is therefore tied to a closed | |
| # one — calls would raise "Event loop is closed". Re-init each | |
| # invocation; the orphaned async client gets garbage-collected. | |
| from app.core.redis import init_redis | |
| await init_redis() | |
| # Read from IFC processing stream | |
| messages = await RedisStream.read_events( | |
| stream="ifc.process", | |
| consumer_group="ifc_processors", | |
| consumer_name="worker-1", | |
| count=10, | |
| block=1000 | |
| ) | |
| for stream, msgs in messages: | |
| for msg_id, fields in msgs: | |
| try: | |
| # Extract project data from stream event | |
| project_data = {} | |
| for i in range(0, len(fields), 2): | |
| key = fields[i].decode() if isinstance(fields[i], bytes) else fields[i] | |
| value = fields[i + 1].decode() if isinstance(fields[i + 1], bytes) else fields[i + 1] | |
| project_data[key] = value | |
| project_id = project_data.get("project_id") | |
| logger.info(f"Processing IFC stream event for project {project_id}") | |
| # Trigger the actual processing task | |
| process_ifc_file.delay(project_data) | |
| processed_count += 1 | |
| # Acknowledge the message | |
| await RedisStream.ack_event("ifc.process", "ifc_processors", msg_id) | |
| except Exception as e: | |
| logger.error(f"Error processing IFC stream message {msg_id}: {e}") | |
| continue | |
| logger.info(f"Processed {processed_count} IFC events from stream") | |
| return {"processed": processed_count} | |
| except Exception as e: | |
| logger.error(f"Error processing IFC stream: {e}") | |
| return {"error": str(e)} |