bimquery-api / app /api /v1 /endpoints /query.py
BIM Query Deploy
Claude Opus 4.7 (1M context)
deploy: initial BIM Query API on Hugging Face Spaces
e6be672
Raw History Blame Contribute Delete
47.5 kB
# Copyright (C) 2026 Mattia Zatelli # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. """ Endpoints for natural language queries on BIM projects These endpoints handle the entire query processing pipeline: - Natural language interpretation via LLM - Intent classification and entity extraction - Conversion to structured operations - Database execution with optimizations - Human-readable response generation - Intelligent caching for performance - Batch support for multiple queries Supports complex multi-intent queries with advanced semantic analysis. """ import logging import hashlib from typing import List, Optional from datetime import datetime from fastapi import APIRouter, Depends, HTTPException, status, Request, BackgroundTasks from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, and_, or_, desc from app.core.database import get_async_session from app.models import Project, ProjectStatus, Query, QueryType, QueryStatus from app.schemas.query import ( QueryRequest, QueryResponse, QueryBatchRequest, QueryBatchResponse, CalculateRequest, CalculateResponse, QueryListRequest, QueryListResponse, QueryFeedbackRequest, QueryFeedbackResponse ) from app.schemas.common import ErrorResponse, SuccessResponse from app.core.exceptions import ProjectNotFoundError, ProjectNotReadyError, QueryError from app.core.redis import RedisCache logger = logging.getLogger(__name__) router = APIRouter() async def get_project_for_query( project_id: str, session: AsyncSession, tenant_id: str ) -> Project: """Retrieve and validate a project for query operations Verifies that the project is accessible to the tenant and ready for queries. Also updates the last access timestamp for the tiered storage system. Args: project_id: External project ID format (prj_123) session: Async database session tenant_id: Tenant ID for isolation Returns: Project: Validated project ready for queries Raises: ProjectNotFoundError: If project not found ProjectNotReadyError: If project not yet processed """ # Convert external ID to internal format try: internal_id = int(project_id.replace('prj_', '')) except ValueError: raise ProjectNotFoundError(project_id) # Query with multi-tenant isolation query = select(Project).where( and_( Project.id == internal_id, Project.tenant_id == tenant_id # Security: only tenant's projects ) ) result = await session.execute(query) project = result.scalar_one_or_none() if not project: raise ProjectNotFoundError(project_id) # Verify that the project is ready for queries if not project.is_ready(): raise ProjectNotReadyError(project_id, project.status.value) # Update access timestamp for tiered storage management project.touch_last_access() await session.flush() return project def generate_query_hash(query_text: str, project_id: str, options: dict = None) -> str: """Generate a unique hash for query caching The hash is used for: - Caching identical query results - Deduplicating multiple requests - Selective cache invalidation Args: query_text: Original query text project_id: Project ID options: Additional query options Returns: str: SHA256 hash of the query """ content = f"{query_text}:{project_id}" if options: # Sort options for consistent hash content += f":{str(sorted(options.items()))}" return hashlib.sha256(content.encode()).hexdigest() async def check_query_cache(query_hash: str) -> Optional[dict]: """Check if the query result exists in cache Args: query_hash: Query hash generated by generate_query_hash() Returns: dict: Cached result or None if not found """ cache_key = f"query_result:{query_hash}" return await RedisCache.get(cache_key) async def cache_query_result(query_hash: str, result: dict, ttl: int = 3600): """Cache a query result Args: query_hash: Unique query hash result: Result to cache ttl: Time-to-live in seconds (default 1 hour) """ cache_key = f"query_result:{query_hash}" await RedisCache.set(cache_key, result, ttl) # Removed _should_use_fast_mode - always use full LLM synthesis for quality @router.post("", response_model=QueryResponse) async def process_query( request_data: QueryRequest, request: Request, background_tasks: BackgroundTasks, session: AsyncSession = Depends(get_async_session) ): """ Process a natural language query **Processing pipeline:** 1. **Validation and Cache Check** - Verify project and permissions - Check cache for existing results 2. **Linguistic Analysis (NLP)** - Intent classification via LLM - Entity and parameter extraction - Multi-intent query decomposition 3. **Query Planning** - Conversion to structured operations - Database optimization - Sub-query dependency management 4. **Execution** - Database query with optimized indexes - Geometric calculations (CPU/GPU) - Aggregations and post-processing 5. **Result Synthesis** - Human-readable explanation generation - Presentation formatting - Cache for reuse **Supported query examples:** - "Quante finestre ci sono al primo piano?" - "Calcola l'area totale dei muri esterni" - "Trova tutti gli elementi non conformi REI 60" - "Distanza minima tra scale di emergenza" Args: request_data: Query request data Returns: QueryResponse: Processing result with explanation Raises: ProjectNotFoundError: Project not found ProjectNotReadyError: Project not processed QueryError: Error during query processing """ # Extract authentication info from middleware tenant_id = getattr(request.state, "tenant_id", "unknown") api_key_hash = getattr(request.state, "api_key_hash", None) request_id = getattr(request.state, "request_id", "unknown") logger.info(f"Starting query processing for tenant {tenant_id}: {request_data.query[:100]}") # === PROJECT VALIDATION === # Verify that the project exists, is accessible and ready for queries project = await get_project_for_query(request_data.project_id, session, tenant_id) # Generate query hash for caching query_hash = generate_query_hash( request_data.query, request_data.project_id, request_data.options ) # Check cache first cached_result = await check_query_cache(query_hash) if cached_result: logger.info(f"Query cache hit for tenant {tenant_id}") # Create query record for tracking query_record = Query( project_id=project.id, tenant_id=tenant_id, api_key_hash=api_key_hash, query_type=QueryType.NATURAL_LANGUAGE, original_query=request_data.query, query_hash=query_hash, status=QueryStatus.CACHED, cache_hit=True, processing_time_ms=0 ) session.add(query_record) await session.flush() await session.refresh(query_record) # Return cached response # Filter provenance based on include_provenance flag if not request_data.include_provenance: cached_result["provenance"] = [] cached_result.update({ "query_id": f"qry_{query_record.id}", "cache_hit": True }) return QueryResponse(**cached_result) try: # Create query record query_record = Query( project_id=project.id, tenant_id=tenant_id, api_key_hash=api_key_hash, query_type=QueryType.NATURAL_LANGUAGE, original_query=request_data.query, processed_query=request_data.query.strip(), query_hash=query_hash, status=QueryStatus.PROCESSING, query_options=request_data.options ) session.add(query_record) await session.flush() await session.refresh(query_record) query_public_id = f"qry_{query_record.id}" # Process natural language query processing_start = datetime.now() # Initialize timing tracking timing_breakdown = {} # Use enhanced NLP processor for intent classification and query planning from app.services.enhanced_nlp_processor import enhanced_nlp_processor from app.core.intent_system.plugin_registry import plugin_dispatcher from app.services.response_synthesis_service import response_synthesis_service from app.services.result_enhancement_service import result_enhancement_service # Step 1: Intent classification and query planning step1_start = datetime.now() classification_result = await enhanced_nlp_processor.classify_intent( request_data.query, context={"project_id": project.id, "tenant_id": tenant_id} ) step1_end = datetime.now() timing_breakdown["intent_classification_ms"] = int((step1_end - step1_start).total_seconds() * 1000) # Step 2: Process query using intent handlers step2_start = datetime.now() # Add include_provenance flag to options for handlers handler_options = request_data.options or {} handler_options["include_provenance"] = request_data.include_provenance basic_query_result = await process_multi_intent_query( classification_result.query_plan, project, session, handler_options ) step2_end = datetime.now() timing_breakdown["query_execution_ms"] = int((step2_end - step2_start).total_seconds() * 1000) # Step 3: Enhance results with confidence and provenance step3_start = datetime.now() enhanced_result = await result_enhancement_service.enhance_query_result( basic_query_result, project, request_data.query, classification_result.query_plan ) step3_end = datetime.now() timing_breakdown["result_enhancement_ms"] = int((step3_end - step3_start).total_seconds() * 1000) # Step 4: Synthesize human-readable response step4_start = datetime.now() # Always use full LLM synthesis for best quality synthesis_result = await response_synthesis_service.synthesize_response( basic_query_result, request_data.query ) step4_end = datetime.now() timing_breakdown["response_synthesis_ms"] = int((step4_end - step4_start).total_seconds() * 1000) # Step 5: Result assembly and formatting step5_start = datetime.now() # Combine enhanced result with synthesis result = { "result": synthesis_result["synthesized_response"], "confidence": enhanced_result.overall_confidence, "explanation": enhanced_result.overall_explanation, "details": { "structured_plan": { "sub_queries": [ { "intent": sq.intent.value, "subtype": sq.subtype, "query_fragment": sq.query_fragment, "filter_groups": [fg.dict() for fg in (sq.filter_groups or [])], "scope": sq.scope.dict() if sq.scope else None, "confidence": sq.confidence } for sq in classification_result.query_plan.sub_queries ], "is_multi_intent": classification_result.query_plan.is_multi_intent, "intent_relationships": classification_result.query_plan.intent_relationships }, "elements_counted": enhanced_result.total_elements_processed, "elements_matched": enhanced_result.total_elements_processed, "processing_steps": ["intent_classification", "query_planning", "intent_handling", "result_enhancement", "response_synthesis"], "optimizations_applied": ["intent_registry", "confidence_scoring", "provenance_tracking"], "query_structure": { "main_intent": classification_result.main_intent.value, "sub_intent": classification_result.sub_intent, "confidence": classification_result.confidence }, "data_quality": { "score": enhanced_result.data_quality_score, "notes": enhanced_result.data_quality_notes, "uncertainty_factors": enhanced_result.uncertainty_factors } }, "provenance": enhanced_result.all_provenance if request_data.include_provenance else [], "usage": { "queries": 1, "tokens": classification_result.tokens_used + synthesis_result["synthesis_metadata"]["tokens_used"], "compute_ms": enhanced_result.total_processing_time_ms } } # Complete timing tracking for result assembly phase step5_end = datetime.now() timing_breakdown["result_assembly_ms"] = int((step5_end - step5_start).total_seconds() * 1000) processing_end = datetime.now() processing_time_ms = int((processing_end - processing_start).total_seconds() * 1000) # Add overall timing timing_breakdown["total_processing_ms"] = processing_time_ms # Add timing breakdown to result details result["details"]["timing_breakdown"] = timing_breakdown # Update query record with enhanced results query_record.status = QueryStatus.COMPLETED query_record.processing_time_ms = processing_time_ms query_record.result_summary = result["result"] query_record.result_explanation = result["explanation"] query_record.confidence_score = result["confidence"] # Enhanced fields for multi-intent support query_record.plain_text_prompt = request_data.query # User's original query in plain text query_record.is_multi_intent = classification_result.query_plan.is_multi_intent # Store sub-query information sub_queries_data = [] for sr in basic_query_result.sub_results: sub_queries_data.append({ "intent": sr.intent.value, "subtype": sr.subtype, "query_fragment": sr.query_fragment, "confidence": sr.confidence, "result": sr.result, "elements_processed": sr.elements_processed, "processing_time_ms": sr.processing_time_ms, "provenance": sr.provenance[:10] # Limit to avoid huge JSON }) query_record.sub_queries = sub_queries_data # Store intent relationships query_record.intent_relationships = classification_result.query_plan.intent_relationships # Legacy fields for backward compatibility primary_intent = classification_result.main_intent.value query_record.intent = primary_intent query_record.entities = [sq.intent.value for sq in classification_result.query_plan.sub_queries] # Aggregate filters from all sub-queries for legacy compatibility aggregated_filters = {} for sq in classification_result.query_plan.sub_queries: if sq.filter_groups: # Convert filter_groups to legacy dict format for fg in sq.filter_groups: for f in fg.filters: key, value = f.key, f.val if key in aggregated_filters: # Handle multiple values for the same filter key existing = aggregated_filters[key] if isinstance(existing, list): if isinstance(value, list): aggregated_filters[key] = existing + value else: aggregated_filters[key].append(value) else: if isinstance(value, list): aggregated_filters[key] = [existing] + value else: aggregated_filters[key] = [existing, value] else: aggregated_filters[key] = value query_record.filters = aggregated_filters query_record.result_count = result["details"]["elements_counted"] query_record.completed_at = processing_end await session.commit() # Cache the result cache_data = { "project_id": request_data.project_id, "original_query": request_data.query, "query_type": QueryType.NATURAL_LANGUAGE, "status": QueryStatus.COMPLETED, "result": result["result"], "confidence": result["confidence"], "explanation": result["explanation"], "details": result["details"], "provenance": result["provenance"], "usage": result["usage"], "processing_time_ms": processing_time_ms, "created_at": query_record.created_at.isoformat(), "completed_at": processing_end.isoformat() } await cache_query_result(query_hash, cache_data) # Update project timestamps (both query-specific and general access) project.update_query_timestamp() project.touch_last_access() logger.info(f"Query {query_public_id} completed in {processing_time_ms}ms") return QueryResponse( query_id=query_public_id, project_id=request_data.project_id, original_query=request_data.query, query_type=QueryType.NATURAL_LANGUAGE, status=QueryStatus.COMPLETED, result=result["result"], confidence=result["confidence"], explanation=result["explanation"], details=result["details"], provenance=result["provenance"], usage=result["usage"], processing_time_ms=processing_time_ms, cache_hit=False, created_at=query_record.created_at, completed_at=processing_end ) except Exception as e: # Mark query as failed if 'query_record' in locals(): try: await session.rollback() query_record.status = QueryStatus.FAILED query_record.error_message = str(e)[:500] await session.commit() except Exception: pass # Best effort logger.error(f"Query processing failed for tenant {tenant_id}: {e}") raise QueryError("Query processing failed. Please try again.") async def process_multi_intent_query( query_plan, project: Project, session: AsyncSession, options: dict ): """ Process a multi-intent query using the intent handler registry Args: query_plan: QueryPlan with sub-queries to process project: BIM project session: Database session options: Query options Returns: QueryResult with all sub-results """ from app.core.intent_system.plugin_registry import plugin_dispatcher from app.schemas.intent import QueryResult, SingleQueryResult import time processing_start = time.time() sub_results = [] # Process each sub-query using appropriate handler for sub_query in query_plan.sub_queries: try: sub_query_start = time.time() # Process the single query plan via plugin dispatcher single_result = await plugin_dispatcher.dispatch( sub_query, project, session, context=options ) sub_query_end = time.time() single_result.processing_time_ms = int((sub_query_end - sub_query_start) * 1000) sub_results.append(single_result) except Exception as e: logger.error(f"Error processing sub-query {sub_query.intent.value}: {e}") # Create error result error_result = SingleQueryResult( intent=sub_query.intent, subtype=sub_query.subtype, query_fragment=sub_query.query_fragment, result=f"Error processing {sub_query.intent.value} query", confidence=0.0, explanation=f"Processing failed for {sub_query.intent.value} query", provenance=[], elements_processed=0, processing_time_ms=0, limitations=["An internal error occurred"] ) sub_results.append(error_result) processing_end = time.time() total_processing_time = int((processing_end - processing_start) * 1000) # Calculate overall metrics total_elements = sum(sr.elements_processed for sr in sub_results) overall_confidence = sum(sr.confidence for sr in sub_results) / len(sub_results) if sub_results else 0.0 return QueryResult( sub_results=sub_results, is_multi_intent=query_plan.is_multi_intent, overall_confidence=overall_confidence, total_processing_time_ms=total_processing_time, total_elements_processed=total_elements, intent_relationships=query_plan.intent_relationships ) @router.post("/batch", response_model=QueryBatchResponse) async def process_batch_queries( request_data: QueryBatchRequest, request: Request, background_tasks: BackgroundTasks, session: AsyncSession = Depends(get_async_session) ): """ Process multiple queries in batch """ tenant_id = getattr(request.state, "tenant_id", "unknown") api_key_hash = getattr(request.state, "api_key_hash", None) request_id = getattr(request.state, "request_id", "unknown") logger.info(f"Batch query processing started for tenant {tenant_id}: {len(request_data.queries)} queries") # Validate project project = await get_project_for_query(request_data.project_id, session, tenant_id) batch_start = datetime.now() results = [] successful_count = 0 failed_count = 0 total_usage = {"queries": 0, "tokens": 0, "compute_ms": 0} # Process each query for i, query_text in enumerate(request_data.queries): try: # Create individual query request query_req = QueryRequest( project_id=request_data.project_id, query=query_text, options=request_data.options or {} ) # Use enhanced NLP processor for batch queries from app.services.enhanced_nlp_processor import enhanced_nlp_processor # Classify intent and process classification_result = await enhanced_nlp_processor.classify_intent( query_text, context={"project_id": project.id, "tenant_id": tenant_id} ) basic_query_result = await process_multi_intent_query( classification_result.query_plan, project, session, request_data.options or {} ) # Create simplified result for batch processing processed_result = { "result": basic_query_result.sub_results[0].result if basic_query_result.sub_results else "No results", "confidence": basic_query_result.overall_confidence, "explanation": basic_query_result.sub_results[0].explanation if basic_query_result.sub_results else "", "provenance": basic_query_result.sub_results[0].provenance if basic_query_result.sub_results else [], "usage": { "queries": 1, "tokens": classification_result.tokens_used, "compute_ms": basic_query_result.total_processing_time_ms } } # Create query record query_record = Query( project_id=project.id, tenant_id=tenant_id, api_key_hash=api_key_hash, query_type=QueryType.BATCH, original_query=query_text, query_hash=generate_query_hash(query_text, request_data.project_id), status=QueryStatus.COMPLETED, result_summary=processed_result["result"], confidence_score=processed_result["confidence"], processing_time_ms=processed_result["usage"]["compute_ms"] ) session.add(query_record) await session.flush() results.append({ "query": query_text, "status": QueryStatus.COMPLETED, "result": processed_result["result"], "confidence": processed_result["confidence"], "explanation": processed_result["explanation"], "provenance": processed_result["provenance"], "processing_time_ms": processed_result["usage"]["compute_ms"] }) successful_count += 1 for key, value in processed_result["usage"].items(): total_usage[key] = total_usage.get(key, 0) + value # Stop on error if requested if request_data.stop_on_error and failed_count > 0: break except Exception as e: logger.error(f"Batch query failed for item '{query_text[:80]}': {e}") results.append({ "query": query_text, "status": QueryStatus.FAILED, "error": "Query processing failed", "provenance": [], "processing_time_ms": None }) failed_count += 1 if request_data.stop_on_error: break batch_end = datetime.now() total_time = int((batch_end - batch_start).total_seconds() * 1000) await session.commit() logger.info(f"Batch query completed: {successful_count} successful, {failed_count} failed") return QueryBatchResponse( batch_id=f"batch_{request_id}", project_id=request_data.project_id, total_queries=len(request_data.queries), successful=successful_count, failed=failed_count, results=results, total_processing_time_ms=total_time, parallel_execution=request_data.parallel_execution, usage=total_usage, created_at=batch_start, completed_at=batch_end ) @router.get("", response_model=QueryListResponse) async def list_queries( request: Request, project_id: Optional[str] = None, query_type: Optional[QueryType] = None, status: Optional[QueryStatus] = None, search: Optional[str] = None, limit: int = 50, cursor: Optional[str] = None, session: AsyncSession = Depends(get_async_session) ): """ List queries for the authenticated tenant """ tenant_id = getattr(request.state, "tenant_id", "unknown") # Build query query = select(Query).where(Query.tenant_id == tenant_id) # Apply filters if project_id: try: internal_project_id = int(project_id.replace('prj_', '')) query = query.where(Query.project_id == internal_project_id) except ValueError: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid project ID format" ) if query_type: query = query.where(Query.query_type == query_type) if status: query = query.where(Query.status == status) if search: search_term = f"%{search.lower()}%" query = query.where(Query.original_query.ilike(search_term)) # Apply cursor pagination if cursor: try: cursor_id = int(cursor) query = query.where(Query.id > cursor_id) except ValueError: pass # Apply limit and ordering query = query.order_by(desc(Query.created_at)).limit(limit + 1) # Execute query result = await session.execute(query) queries = result.scalars().all() # Check for more results has_next = len(queries) > limit if has_next: queries = queries[:-1] # Generate next cursor next_cursor = str(queries[-1].id) if has_next and queries else None # Convert to response format query_items = [] for q in queries: query_items.append({ "query_id": f"qry_{q.id}", "original_query": q.original_query, "query_type": q.query_type, "status": q.status, "result": q.result_summary, "confidence": q.confidence_score, "processing_time_ms": q.processing_time_ms, "created_at": q.created_at, "user_feedback": q.user_feedback }) return QueryListResponse( queries=query_items, pagination={ "limit": limit, "cursor": cursor, "has_next": has_next, "next_cursor": next_cursor, "total_count": None } ) @router.post("/enhanced", response_model=dict) async def process_enhanced_query( request_data: QueryRequest, request: Request, background_tasks: BackgroundTasks, session: AsyncSession = Depends(get_async_session) ): """ Process a natural language query with enhanced multi-intent support and detailed metadata This endpoint returns the full enhanced result structure including: - Detailed confidence breakdown - Provenance metadata - Data quality assessment - Actionable insights - Uncertainty factors """ # Get authentication info tenant_id = getattr(request.state, "tenant_id", "unknown") api_key_hash = getattr(request.state, "api_key_hash", None) request_id = getattr(request.state, "request_id", "unknown") logger.info(f"Enhanced query processing started for tenant {tenant_id}: {request_data.query[:100]}") # Validate project project = await get_project_for_query(request_data.project_id, session, tenant_id) try: # Import services from app.services.enhanced_nlp_processor import enhanced_nlp_processor from app.services.response_synthesis_service import response_synthesis_service from app.services.result_enhancement_service import result_enhancement_service processing_start = datetime.now() # Step 1: Intent classification and query planning classification_result = await enhanced_nlp_processor.classify_intent( request_data.query, context={"project_id": project.id, "tenant_id": tenant_id} ) # Step 2: Process query using intent handlers basic_query_result = await process_multi_intent_query( classification_result.query_plan, project, session, request_data.options or {} ) # Step 3: Enhance results with confidence and provenance enhanced_result = await result_enhancement_service.enhance_query_result( basic_query_result, project, request_data.query, classification_result.query_plan ) # Step 4: Synthesize human-readable response # Always use full LLM synthesis for best quality synthesis_result = await response_synthesis_service.synthesize_response( basic_query_result, request_data.query ) processing_end = datetime.now() processing_time_ms = int((processing_end - processing_start).total_seconds() * 1000) # Create query record query_record = Query( project_id=project.id, tenant_id=tenant_id, api_key_hash=api_key_hash, query_type=QueryType.NATURAL_LANGUAGE, original_query=request_data.query, plain_text_prompt=request_data.query, processed_query=request_data.query.strip(), query_hash=generate_query_hash(request_data.query, request_data.project_id), status=QueryStatus.COMPLETED, is_multi_intent=classification_result.query_plan.is_multi_intent, result_summary=synthesis_result["synthesized_response"], result_explanation=enhanced_result.overall_explanation, confidence_score=enhanced_result.overall_confidence, intent=classification_result.main_intent.value, processing_time_ms=processing_time_ms, result_count=enhanced_result.total_elements_processed, completed_at=processing_end, query_options=request_data.options ) session.add(query_record) await session.flush() await session.refresh(query_record) query_public_id = f"qry_{query_record.id}" await session.commit() # Update project timestamps project.update_query_timestamp() project.touch_last_access() logger.info(f"Enhanced query {query_public_id} completed in {processing_time_ms}ms") # Return comprehensive enhanced result return { "query_id": query_public_id, "project_id": request_data.project_id, "original_query": request_data.query, "query_type": QueryType.NATURAL_LANGUAGE, "status": QueryStatus.COMPLETED, # Core results "synthesized_response": synthesis_result["synthesized_response"], "overall_confidence": enhanced_result.overall_confidence, "overall_explanation": enhanced_result.overall_explanation, # Multi-intent structure "is_multi_intent": enhanced_result.is_multi_intent, "sub_results": [ { "intent": sr.intent.value, "subtype": sr.subtype, "query_fragment": sr.query_fragment, "result": sr.result, "confidence": sr.confidence, "explanation": sr.explanation, "provenance": sr.provenance, "elements_processed": sr.elements_processed, "processing_time_ms": sr.processing_time_ms, "limitations": sr.limitations, "result_data": sr.result_data } for sr in enhanced_result.sub_results ], # Enhanced metadata "confidence_breakdown": enhanced_result.confidence_breakdown, "provenance_metadata": enhanced_result.provenance_metadata, "data_quality": { "score": enhanced_result.data_quality_score, "notes": enhanced_result.data_quality_notes, "has_issues": enhanced_result.has_data_quality_issues }, "uncertainty_factors": enhanced_result.uncertainty_factors, # Actionable insights "next_steps": enhanced_result.next_steps, "related_queries": enhanced_result.related_queries, # Performance metrics "processing_metrics": { "total_processing_time_ms": processing_time_ms, "total_elements_processed": enhanced_result.total_elements_processed, "query_complexity_score": enhanced_result.query_complexity_score, "classification_time_ms": classification_result.classification_time_ms, "synthesis_tokens": synthesis_result["synthesis_metadata"]["tokens_used"] }, # Intent classification details "classification_details": { "main_intent": classification_result.main_intent.value, "sub_intent": classification_result.sub_intent, "classification_confidence": classification_result.confidence, "reasoning": classification_result.reasoning, "llm_model": classification_result.llm_model }, # Usage tracking "usage": { "queries": 1, "tokens": total_tokens, "compute_ms": processing_time_ms }, "created_at": query_record.created_at, "completed_at": processing_end, "cache_hit": False } except Exception as e: # Create failed query record if possible if 'query_record' in locals(): query_record.status = QueryStatus.FAILED query_record.error_message = str(e) await session.commit() logger.error(f"Enhanced query processing failed for tenant {tenant_id}: {e}") raise QueryError("Query processing failed. Please try again.") @router.get("/intents", response_model=dict) async def get_supported_intents(request: Request): """ Get information about all supported intents and sub-intents """ from app.core.intent_system.plugin_registry import plugin_registry statistics = plugin_registry.get_statistics() all_metadata = plugin_registry.get_all_metadata() # Build intent descriptions from manifests intent_descriptions = { meta.id: meta.description for meta in all_metadata } # Build sub-intent descriptions from manifests subintent_descriptions = {} for meta in all_metadata: for si_key, si_meta in meta.sub_intents.items(): subintent_descriptions[si_meta.id] = si_meta.description return { "supported_intents": [m.id for m in all_metadata], "intent_descriptions": intent_descriptions, "subintent_descriptions": subintent_descriptions, "handler_statistics": statistics, } @router.get("/intents/{intent_id}", response_model=dict) async def get_intent_details(intent_id: str, request: Request): """Get detailed information about a specific intent plugin (marketplace).""" from app.core.intent_system.plugin_registry import plugin_registry meta = plugin_registry.metadata.get(intent_id) if not meta: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Intent '{intent_id}' not found", ) return { "id": meta.id, "version": meta.version, "name": meta.name, "description": meta.description, "category": meta.category, "enabled": meta.enabled, "priority": meta.priority, "author": {"name": meta.author.name, "url": meta.author.url}, "sub_intents": { key: { "id": si.id, "description": si.description, "examples": si.examples, "handler_method": si.handler_method, } for key, si in meta.sub_intents.items() }, "i18n": meta.i18n, "prompt_fragment": meta.prompt_fragment, } @router.post("/intents/reload", response_model=dict) async def reload_intents(request: Request): """Hot-reload all intent plugins without server restart.""" from app.core.intent_system.plugin_registry import ( plugin_registry, dynamic_prompt_builder, ) result = plugin_registry.reload() # Rebuild the LLM prompt with updated plugins from app.services.enhanced_nlp_processor import enhanced_nlp_processor enhanced_nlp_processor.system_prompt = dynamic_prompt_builder.build_system_prompt() logger.info(f"Intent plugins reloaded: {result}") return {"status": "reloaded", "changes": result} @router.post("/intents/{intent_id}/disable", response_model=dict) async def disable_intent(intent_id: str, request: Request): """Disable an intent plugin at runtime.""" from app.core.intent_system.plugin_registry import plugin_registry success = plugin_registry.disable_plugin(intent_id) if not success: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Intent '{intent_id}' not found or already disabled", ) return {"status": "disabled", "intent_id": intent_id} @router.post("/intents/{intent_id}/enable", response_model=dict) async def enable_intent(intent_id: str, request: Request): """Re-enable a previously disabled intent plugin.""" from app.core.intent_system.plugin_registry import plugin_registry success = plugin_registry.enable_plugin(intent_id) if not success: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Intent '{intent_id}' not found or failed to enable", ) return {"status": "enabled", "intent_id": intent_id} @router.post("/intents/{intent_id}/validate", response_model=dict) async def validate_intent_manifest(intent_id: str, request: Request): """Validate an intent plugin's manifest.yaml against the schema.""" from app.core.intent_system.manifest_validator import manifest_validator from pathlib import Path intents_dir = Path(__file__).parent.parent.parent.parent / "intents" # Check core and community directories manifest_path = intents_dir / intent_id / "manifest.yaml" is_community = False if not manifest_path.exists(): manifest_path = intents_dir / "_community" / intent_id / "manifest.yaml" is_community = True if not manifest_path.exists(): raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Manifest not found for intent '{intent_id}'", ) if is_community: is_valid, errors = manifest_validator.validate_community_plugin(manifest_path) else: is_valid, errors = manifest_validator.validate_file(manifest_path) return { "intent_id": intent_id, "is_valid": is_valid, "is_community": is_community, "errors": [ {"field": e.field, "message": e.message, "severity": e.severity} for e in errors ], } @router.post("/plan", response_model=dict) async def plan_query( request_data: QueryRequest, request: Request, session: AsyncSession = Depends(get_async_session) ): """ Plan a query without executing it - useful for understanding what the system will do """ tenant_id = getattr(request.state, "tenant_id", "unknown") # Validate project project = await get_project_for_query(request_data.project_id, session, tenant_id) try: from app.services.enhanced_nlp_processor import enhanced_nlp_processor # Classify intent and create query plan classification_result = await enhanced_nlp_processor.classify_intent( request_data.query, context={"project_id": project.id, "tenant_id": tenant_id} ) return { "original_query": request_data.query, "classification": { "main_intent": classification_result.main_intent.value, "sub_intent": classification_result.sub_intent, "confidence": classification_result.confidence, "reasoning": classification_result.reasoning, "llm_model": classification_result.llm_model, "classification_time_ms": classification_result.classification_time_ms, "tokens_used": classification_result.tokens_used }, "query_plan": { "is_multi_intent": classification_result.query_plan.is_multi_intent, "sub_queries": [ { "intent": sq.intent.value, "subtype": sq.subtype, "query_fragment": sq.query_fragment, "confidence": sq.confidence, "filter_groups": [fg.dict() for fg in (sq.filter_groups or [])], "scope": sq.scope.dict() if sq.scope else None, "spatial_params": sq.spatial_params, "compliance_rules": sq.compliance_rules, "insight_params": sq.insight_params } for sq in classification_result.query_plan.sub_queries ], "intent_relationships": classification_result.query_plan.intent_relationships, "presentation_hint": classification_result.query_plan.presentation_hint.value, "confidence_threshold": classification_result.query_plan.confidence_threshold, "max_results": classification_result.query_plan.max_results }, "alternatives": [ { "is_multi_intent": alt.is_multi_intent, "primary_intent": alt.primary_intent.value, "sub_queries_count": len(alt.sub_queries) } for alt in classification_result.alternatives ] } except Exception as e: logger.error(f"Query planning failed: {e}") raise QueryError("Query planning failed. Please try again.") @router.post("/{query_id}/feedback", response_model=QueryFeedbackResponse) async def submit_query_feedback( query_id: str, feedback: QueryFeedbackRequest, request: Request, session: AsyncSession = Depends(get_async_session) ): """ Submit feedback for a query """ tenant_id = getattr(request.state, "tenant_id", "unknown") # Parse and validate query ID try: internal_id = int(query_id.replace('qry_', '')) except ValueError: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid query ID format" ) # Find query query = select(Query).where( and_( Query.id == internal_id, Query.tenant_id == tenant_id ) ) result = await session.execute(query) query_record = result.scalar_one_or_none() if not query_record: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Query not found" ) # Update feedback query_record.set_user_feedback(feedback.rating, feedback.comments) await session.commit() logger.info(f"Feedback submitted for query {query_id}: rating {feedback.rating}") return QueryFeedbackResponse( query_id=query_id, feedback_recorded=True, message="Feedback recorded successfully" )
# Copyright (C) 2026 Mattia Zatelli
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Endpoints for natural language queries on BIM projects
These endpoints handle the entire query processing pipeline:
- Natural language interpretation via LLM
- Intent classification and entity extraction
- Conversion to structured operations
- Database execution with optimizations
- Human-readable response generation
- Intelligent caching for performance
- Batch support for multiple queries
Supports complex multi-intent queries with advanced semantic analysis.
"""
import logging
import hashlib
from typing import List, Optional
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, status, Request, BackgroundTasks
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_, or_, desc
from app.core.database import get_async_session
from app.models import Project, ProjectStatus, Query, QueryType, QueryStatus
from app.schemas.query import (
QueryRequest, QueryResponse, QueryBatchRequest, QueryBatchResponse,
CalculateRequest, CalculateResponse, QueryListRequest, QueryListResponse,
QueryFeedbackRequest, QueryFeedbackResponse
)
from app.schemas.common import ErrorResponse, SuccessResponse
from app.core.exceptions import ProjectNotFoundError, ProjectNotReadyError, QueryError
from app.core.redis import RedisCache
logger = logging.getLogger(__name__)
router = APIRouter()
async def get_project_for_query(
project_id: str,
session: AsyncSession,
tenant_id: str
) -> Project:
"""Retrieve and validate a project for query operations
Verifies that the project is accessible to the tenant and ready for queries.
Also updates the last access timestamp for the tiered storage system.
Args:
project_id: External project ID format (prj_123)
session: Async database session
tenant_id: Tenant ID for isolation
Returns:
Project: Validated project ready for queries
Raises:
ProjectNotFoundError: If project not found
ProjectNotReadyError: If project not yet processed
"""
# Convert external ID to internal format
try:
internal_id = int(project_id.replace('prj_', ''))
except ValueError:
raise ProjectNotFoundError(project_id)
# Query with multi-tenant isolation
query = select(Project).where(
and_(
Project.id == internal_id,
Project.tenant_id == tenant_id # Security: only tenant's projects
)
)
result = await session.execute(query)
project = result.scalar_one_or_none()
if not project:
raise ProjectNotFoundError(project_id)
# Verify that the project is ready for queries
if not project.is_ready():
raise ProjectNotReadyError(project_id, project.status.value)
# Update access timestamp for tiered storage management
project.touch_last_access()