Buckets:

download
raw
120 kB
import logging import os import sys import threading import time from pathlib import Path from typing import Any, Dict, List, Optional from fastapi import Depends, FastAPI, HTTPException, Request, Response from fastapi.middleware.cors import CORSMiddleware from .artifacts.semantic_profiles import SemanticProfileRegistry from .cache import ( CacheManagerFactory, QueryCacheConfig, QueryResultCache, QueryType, ) from .cli.index_management import _get_profile_collection_name from .cli.preflight_commands import format_preflight_report, run_startup_preflight from .client import ( ClientValidationError, build_search_options, execute_search_service, search_result_for_gateway, ) from .config.environment import is_production as _is_production from .config.settings import get_settings from .config.validation import ( WEAK_CREDENTIAL_BLOCKLIST, render_validation_errors_to_stderr, validate_production_config, ) from .core import RepoContext, RepoResolver from .core.logging import setup_logging from .dispatcher.dispatcher_enhanced import EnhancedDispatcher from .health.repository_readiness import RepositoryReadiness, RepositoryReadinessState from .indexer.bm25_indexer import BM25Indexer from .indexer.hybrid_search import HybridSearch, HybridSearchConfig from .indexing.source_metadata import normalize_friction_category from .metrics import get_health_checker, get_metrics_collector from .metrics.middleware import get_business_metrics, setup_metrics_middleware from .metrics.prometheus_exporter import get_prometheus_exporter from .plugin_base import SymbolDef from .plugin_system import PluginManager from .security import ( AuthCredentials, AuthenticationMiddleware, AuthManager, AuthorizationMiddleware, Permission, RateLimitMiddleware, RequestValidationMiddleware, SecurityConfig, SecurityHeadersMiddleware, User, UserRole, get_current_active_user, require_auth, require_permission, require_role, ) from .security.path_guard import PathTraversalError, PathTraversalGuard from .security.security_middleware import SecretRedactionResponseMiddleware from .security.token_validator import TokenValidator from .setup.qdrant_autostart import ensure_qdrant_running from .setup.semantic_preflight import run_semantic_preflight from .storage.sqlite_store import SQLiteStore from .storage.store_registry import StoreRegistry from .utils.fuzzy_indexer import FuzzyIndexer from .utils.index_discovery import IndexDiscovery from .utils.language_detector import detect_repository_languages from .watcher.ref_poller import RefPoller from .watcher_multi_repo import MultiRepositoryWatcher # Set up logging setup_logging(log_level="INFO") logger = logging.getLogger(__name__) def _parse_cors_origins_from_env() -> list[str]: """Parse CORS_ORIGINS env var into a cleaned list of origins. No default — an unset or empty value means "no cross-origin requests allowed" and emits a warning. Accepts comma-separated origins. Explicitly refuses "*" which would reintroduce the old footgun. """ raw = os.getenv("CORS_ORIGINS", "").strip() if not raw: logger.warning( "CORS_ORIGINS is unset — no cross-origin requests will be allowed. " "Set CORS_ORIGINS to a comma-separated allowlist (never '*' in production)." ) return [] origins = [o.strip() for o in raw.split(",") if o.strip()] if "*" in origins: logger.warning( "CORS_ORIGINS contains '*' — refusing wildcard origin. " "Replace with an explicit allowlist." ) return [o for o in origins if o != "*"] return origins def _build_security_config_from_env() -> SecurityConfig: """Build SecurityConfig from environment and fail closed on invalid input.""" jwt_secret_key = os.getenv("JWT_SECRET_KEY") if not jwt_secret_key: raise RuntimeError( "JWT_SECRET_KEY env var must be set. Generate one with: openssl rand -base64 32" ) try: return SecurityConfig( jwt_secret_key=jwt_secret_key, jwt_algorithm="HS256", access_token_expire_minutes=int(os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", "30")), refresh_token_expire_days=int(os.getenv("REFRESH_TOKEN_EXPIRE_DAYS", "7")), password_min_length=int(os.getenv("PASSWORD_MIN_LENGTH", "8")), max_login_attempts=int(os.getenv("MAX_LOGIN_ATTEMPTS", "5")), lockout_duration_minutes=int(os.getenv("LOCKOUT_DURATION_MINUTES", "15")), rate_limit_requests=int(os.getenv("RATE_LIMIT_REQUESTS", "100")), rate_limit_window_minutes=int(os.getenv("RATE_LIMIT_WINDOW_MINUTES", "1")), cors_origins=_parse_cors_origins_from_env(), cors_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], cors_headers=["*"], ) except Exception as exc: raise RuntimeError(f"Invalid JWT security configuration: {exc}") from exc def _require_default_admin_password() -> str: """Return the configured default admin password or raise fail-closed.""" admin_password = os.getenv("DEFAULT_ADMIN_PASSWORD") if not admin_password: raise RuntimeError( "DEFAULT_ADMIN_PASSWORD env var must be set. Choose a strong password for the admin account." ) if admin_password.strip().lower() in WEAK_CREDENTIAL_BLOCKLIST: raise RuntimeError( "DEFAULT_ADMIN_PASSWORD is on the well-known-weak list. " "Pick a strong password (see config/validation.py blocklist)." ) return admin_password def _build_boot_auth_manager() -> tuple[SecurityConfig | None, AuthManager | None]: """Best-effort import-time auth bootstrap for pre-startup route protection.""" try: config = _build_security_config_from_env() except RuntimeError: return None, None return config, AuthManager(config) def _register_security_middleware( target_app: FastAPI, config: SecurityConfig | None, active_auth_manager: AuthManager | None, ) -> None: """Register the HTTP security stack before startup.""" cors_origins = config.cors_origins if config is not None else _parse_cors_origins_from_env() security_headers = ( config.security_headers if config is not None else SecurityConfig(jwt_secret_key="x" * 32).security_headers ) target_app.add_middleware(SecurityHeadersMiddleware, security_headers=security_headers) target_app.add_middleware(AuthorizationMiddleware, auth_manager=active_auth_manager) target_app.add_middleware(AuthenticationMiddleware, auth_manager=active_auth_manager) target_app.add_middleware(RateLimitMiddleware, auth_manager=active_auth_manager) target_app.add_middleware(RequestValidationMiddleware) target_app.add_middleware( CORSMiddleware, allow_origins=cors_origins, allow_credentials=True, allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], allow_headers=["*"], ) app = FastAPI( title="MCP Server", description="Code Index MCP Server with Security, Metrics, and Health Checks", ) dispatcher: EnhancedDispatcher | None = None repo_resolver: RepoResolver | None = None sqlite_store: SQLiteStore | None = None multi_watcher: MultiRepositoryWatcher | None = None ref_poller: RefPoller | None = None plugin_manager: PluginManager | None = None plugin_loader = None # Dynamic plugin loader auth_manager: AuthManager | None = None security_config: SecurityConfig | None = None cache_manager = None query_cache: QueryResultCache | None = None bm25_indexer: BM25Indexer | None = None hybrid_search: HybridSearch | None = None fuzzy_indexer: FuzzyIndexer | None = None git_index_manager: Any = None semantic_indexer = None profile_hydration_status: Dict[str, Any] | None = None semantic_setup_status: Dict[str, Any] | None = None _repo_registry = None _store_registry: StoreRegistry | None = None security_config, auth_manager = _build_boot_auth_manager() def _get_path_guard() -> Optional[PathTraversalGuard]: from mcp_server.security.path_allowlist import parse_allowed_roots # Read env lazily so tests can monkeypatch MCP_ALLOWED_ROOTS before calling. raw = os.environ.get("MCP_ALLOWED_ROOTS", "") roots = parse_allowed_roots(raw) if not roots: # No roots configured — guard short-circuits (allow-all for dev/test). return None return PathTraversalGuard(roots) def _normalize_search_result(raw_result: Any) -> Optional[dict[str, Any]]: """Normalize internal search payloads to the public SearchResult schema. Returns None if the result's file path fails the path traversal guard. """ def _prefer_path(candidate: Any, fallback: Any) -> str: candidate_str = str(candidate or "") fallback_str = str(fallback or "") if candidate_str and not candidate_str.isdigit(): return candidate_str return fallback_str or candidate_str if isinstance(raw_result, dict): file_value = _prefer_path( raw_result.get("file") or raw_result.get("file_path") or raw_result.get("filepath") or raw_result.get("defined_in"), raw_result.get("relative_path") or raw_result.get("path"), ) guard = _get_path_guard() if guard is not None: try: guard.normalize_and_check(file_value) except PathTraversalError: logger.warning("path traversal attempt blocked: %r", file_value) return None start_line = raw_result.get("start_line") end_line = raw_result.get("end_line") if start_line is None: start_line = raw_result.get("line_start") if end_line is None: end_line = raw_result.get("line_end") if start_line is None: start_line = raw_result.get("line") if start_line is None: span = raw_result.get("span") if isinstance(span, (list, tuple)) and len(span) >= 2: start_line = span[0] if end_line is None: end_line = span[1] elif isinstance(span, (list, tuple)) and len(span) >= 1: start_line = span[0] start_line = int(start_line) if start_line is not None else 1 end_line = int(end_line) if end_line is not None else start_line snippet_value = ( raw_result.get("snippet") or raw_result.get("context") or raw_result.get("content") or raw_result.get("doc") or raw_result.get("signature") or "" ) # Prepend line numbers to snippet if start_line is known snippet_lines = str(snippet_value).split("\n") numbered_snippet = "\n".join( f"{start_line + i}: {line}" for i, line in enumerate(snippet_lines) ) normalized: dict[str, Any] = { "file": str(file_value), "start_line": start_line, "end_line": end_line, "snippet": numbered_snippet, } if raw_result.get("source_metadata") is not None: normalized["source_metadata"] = raw_result.get("source_metadata") return normalized file_value = _prefer_path( getattr(raw_result, "file", None) or getattr(raw_result, "file_path", None) or getattr(raw_result, "filepath", None), getattr(raw_result, "relative_path", None) or getattr(raw_result, "path", None), ) guard = _get_path_guard() if guard is not None: try: guard.normalize_and_check(file_value) except PathTraversalError: logger.warning("path traversal attempt blocked: %r", file_value) return None start_line = ( getattr(raw_result, "start_line", None) or getattr(raw_result, "line_start", None) or getattr(raw_result, "line", None) or 1 ) end_line = ( getattr(raw_result, "end_line", None) or getattr(raw_result, "line_end", None) or start_line ) start_line = int(start_line) end_line = int(end_line) snippet_value = ( getattr(raw_result, "snippet", None) or getattr(raw_result, "context", None) or "" ) snippet_lines = str(snippet_value).split("\n") numbered_snippet = "\n".join( f"{start_line + i}: {line}" for i, line in enumerate(snippet_lines) ) normalized = { "file": str(file_value), "start_line": start_line, "end_line": end_line, "snippet": numbered_snippet, } source_metadata = getattr(raw_result, "source_metadata", None) if source_metadata is not None: normalized["source_metadata"] = source_metadata return normalized def _normalize_source_metadata_categories(value: Optional[str]) -> list[str]: if not value: return [] return [normalize_friction_category(item) for item in value.split(",") if item.strip()] def _normalize_csv_filters(value: Optional[str]) -> list[str]: if not value: return [] return sorted({item.strip() for item in value.split(",") if item.strip()}, key=str.lower) language_detection_status: Dict[str, Any] | None = None _FALLBACK_REPO_ID = "default" _repo_search_backends: dict[str, tuple[SQLiteStore, BM25Indexer, FuzzyIndexer, HybridSearch]] = {} _repo_search_backends_lock = threading.Lock() _RECOVERABLE_REINDEX_STATES = frozenset( { RepositoryReadinessState.MISSING_INDEX, RepositoryReadinessState.INDEX_EMPTY, RepositoryReadinessState.STALE_COMMIT, RepositoryReadinessState.CORRUPT_SQLITE, RepositoryReadinessState.MISSING_SCHEMA, RepositoryReadinessState.MISSING_PROVENANCE, # CHUNKERSAFE Lane A: scheme-mismatched / mid-rebuild indexes must be # reindexable so a tripped guard is not a permanent dead-end. RepositoryReadinessState.SCHEME_MISMATCH, RepositoryReadinessState.INDEX_REBUILDING, } ) def _repository_unavailable_detail(selector: str) -> dict[str, Any]: try: readiness = repo_resolver.classify(selector) if repo_resolver is not None else None except Exception: readiness = None return { "error": "Index unavailable", "code": getattr(getattr(readiness, "state", None), "value", "repository_unavailable"), "safe_fallback": "native_search", "repository": selector, "readiness": readiness.to_dict() if readiness is not None else None, } def _search_backends_for_repo( ctx: RepoContext, ) -> tuple[Optional[BM25Indexer], Optional[FuzzyIndexer], Optional[HybridSearch]]: if ctx.repo_id == _FALLBACK_REPO_ID: return bm25_indexer, fuzzy_indexer, hybrid_search with _repo_search_backends_lock: cached = _repo_search_backends.get(ctx.repo_id) if cached is not None and cached[0] is ctx.sqlite_store: return cached[1], cached[2], cached[3] repo_bm25 = BM25Indexer(ctx.sqlite_store) repo_fuzzy = FuzzyIndexer(ctx.sqlite_store) source_config = hybrid_search.config if hybrid_search is not None else HybridSearchConfig() repo_hybrid = HybridSearch( storage=ctx.sqlite_store, bm25_indexer=repo_bm25, semantic_indexer=None, fuzzy_indexer=repo_fuzzy, config=HybridSearchConfig( bm25_weight=source_config.bm25_weight, semantic_weight=0.0, fuzzy_weight=source_config.fuzzy_weight, rrf_k=source_config.rrf_k, enable_bm25=source_config.enable_bm25, enable_semantic=False, enable_fuzzy=source_config.enable_fuzzy, individual_limit=source_config.individual_limit, final_limit=source_config.final_limit, parallel_execution=source_config.parallel_execution, parallel_timeout_seconds=source_config.parallel_timeout_seconds, cache_results=source_config.cache_results, min_bm25_score=source_config.min_bm25_score, min_semantic_score=source_config.min_semantic_score, min_fuzzy_score=source_config.min_fuzzy_score, ), ) _repo_search_backends[ctx.repo_id] = ( ctx.sqlite_store, repo_bm25, repo_fuzzy, repo_hybrid, ) return repo_bm25, repo_fuzzy, repo_hybrid def get_repo_ctx(request: Request) -> RepoContext: """Resolve a RepoContext for the current request. Resolution order: 1. ``X-Repo-Id`` header value treated as an exact repository selector. 2. ``?repository=<selector>`` query parameter. 3. ``repo_resolver.resolve(cwd)`` fallback when a resolver is available. 4. Minimal context built from the module-global ``sqlite_store`` as last resort. """ candidate_selector: Optional[str] = None repo_id_header = request.headers.get("X-Repo-Id") if repo_id_header: candidate_selector = repo_id_header if candidate_selector is None: candidate_selector = request.query_params.get("repository") if candidate_selector is not None: if repo_resolver is None: raise HTTPException(503, detail=_repository_unavailable_detail(candidate_selector)) try: readiness = repo_resolver.classify(candidate_selector) except Exception as exc: raise HTTPException( 503, detail=_repository_unavailable_detail(candidate_selector) ) from exc if isinstance(readiness, RepositoryReadiness) and not readiness.ready: raise HTTPException( 503, detail={ "error": "Index unavailable", "code": readiness.code, "safe_fallback": "native_search", "repository": candidate_selector, "readiness": readiness.to_dict(), }, ) resolved = ( repo_resolver.resolve_ready(candidate_selector) if isinstance(repo_resolver, RepoResolver) else repo_resolver.resolve(candidate_selector) ) if resolved is not None: return resolved raise HTTPException(503, detail=_repository_unavailable_detail(candidate_selector)) if repo_resolver is not None: cwd_selector = str(Path.cwd()) try: readiness = repo_resolver.classify(cwd_selector) except Exception as exc: raise HTTPException(503, detail=_repository_unavailable_detail(cwd_selector)) from exc if isinstance(readiness, RepositoryReadiness) and not readiness.ready: raise HTTPException( 503, detail={ "error": "Index unavailable", "code": readiness.code, "safe_fallback": "native_search", "readiness": readiness.to_dict(), }, ) resolved = ( repo_resolver.resolve_ready(cwd_selector) if isinstance(repo_resolver, RepoResolver) else repo_resolver.resolve(cwd_selector) ) if resolved is not None: return resolved if isinstance(repo_resolver, RepoResolver): raise HTTPException(503, detail=_repository_unavailable_detail(cwd_selector)) if sqlite_store is None: raise HTTPException(503, detail="Repository resolver and fallback store are unavailable") # Final fallback: wrap the gateway-global sqlite_store in a minimal context. from datetime import datetime from .storage.multi_repo_manager import RepositoryInfo # local import to avoid cycles cwd = Path.cwd() dummy_entry = RepositoryInfo( repository_id=_FALLBACK_REPO_ID, name="default", path=cwd, index_path=cwd, language_stats={}, total_files=0, total_symbols=0, indexed_at=datetime.utcnow(), ) return RepoContext( repo_id=_FALLBACK_REPO_ID, sqlite_store=sqlite_store, # type: ignore[arg-type] workspace_root=cwd, tracked_branch="", registry_entry=dummy_entry, ) # Initialize metrics and health checking metrics_collector = get_metrics_collector() health_checker = get_health_checker() business_metrics = get_business_metrics() # Setup metrics middleware setup_metrics_middleware(app, enable_detailed_metrics=True) # Register SecretRedactionResponseMiddleware at import time so it applies # regardless of how the app is booted. app.add_middleware(SecretRedactionResponseMiddleware) _register_security_middleware(app, security_config, auth_manager) app.state.auth_manager = auth_manager app.state.security_config = security_config @app.on_event("startup") async def startup_event(): """Initialize the dispatcher and register plugins on startup.""" global dispatcher, repo_resolver, sqlite_store, multi_watcher, ref_poller, plugin_manager, plugin_loader, auth_manager, security_config, cache_manager, query_cache, bm25_indexer, hybrid_search, fuzzy_indexer, git_index_manager, semantic_indexer, profile_hydration_status, semantic_setup_status, language_detection_status, _store_registry app.state.startup_time = time.monotonic() try: preflight_result = run_startup_preflight() for line in format_preflight_report(preflight_result): if preflight_result.status == "warning": logger.warning(line) else: logger.info(line) # Initialize security configuration logger.info("Initializing security configuration...") security_config = _build_security_config_from_env() # Validate security config before constructing auth manager _validation_errors = validate_production_config( security_config, environment=str(os.getenv("MCP_ENVIRONMENT", "development")) ) if _validation_errors: render_validation_errors_to_stderr(_validation_errors) if any(e.severity == "fatal" for e in _validation_errors) and _is_production(): sys.exit(1) # Initialize authentication manager logger.info("Initializing authentication manager...") auth_manager = AuthManager(security_config) app.state.auth_manager = auth_manager app.state.security_config = security_config # Validate GitHub token scopes (warns and continues if token absent; raises only if MCP_REQUIRE_TOKEN_SCOPES=1) TokenValidator.validate_scopes( required=[ "contents:read", "metadata:read", "actions:read", "actions:write", "attestations:write", ] ) # Create default admin user if it doesn't exist admin_user = await auth_manager.get_user_by_username("admin") if not admin_user: admin_password = _require_default_admin_password() logger.info("Creating default admin user...") await auth_manager.create_user( username="admin", password=admin_password, email=os.getenv("DEFAULT_ADMIN_EMAIL", "admin@localhost"), role=UserRole.ADMIN, ) logger.info("Default admin user created") # Initialize cache system logger.info("Initializing cache system...") cache_backend_type = os.getenv("CACHE_BACKEND", "memory").lower() redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") if cache_backend_type == "redis": try: cache_manager = CacheManagerFactory.create_redis_cache( redis_url=redis_url, default_ttl=int(os.getenv("CACHE_DEFAULT_TTL", "3600")), ) logger.info("Using Redis cache backend") except Exception as e: logger.warning(f"Failed to initialize Redis cache, falling back to memory: {e}") cache_manager = CacheManagerFactory.create_memory_cache() elif cache_backend_type == "hybrid": try: cache_manager = CacheManagerFactory.create_hybrid_cache( redis_url=redis_url, max_entries=int(os.getenv("CACHE_MAX_ENTRIES", "1000")), max_memory_mb=int(os.getenv("CACHE_MAX_MEMORY_MB", "100")), default_ttl=int(os.getenv("CACHE_DEFAULT_TTL", "3600")), ) logger.info("Using hybrid cache backend") except Exception as e: logger.warning(f"Failed to initialize hybrid cache, falling back to memory: {e}") cache_manager = CacheManagerFactory.create_memory_cache() else: cache_manager = CacheManagerFactory.create_memory_cache( max_entries=int(os.getenv("CACHE_MAX_ENTRIES", "1000")), max_memory_mb=int(os.getenv("CACHE_MAX_MEMORY_MB", "100")), default_ttl=int(os.getenv("CACHE_DEFAULT_TTL", "3600")), ) logger.info("Using memory cache backend") await cache_manager.initialize() # Initialize query result cache query_cache_config = QueryCacheConfig( enabled=os.getenv("QUERY_CACHE_ENABLED", "true").lower() == "true", default_ttl=int(os.getenv("QUERY_CACHE_DEFAULT_TTL", "300")), symbol_lookup_ttl=int(os.getenv("QUERY_CACHE_SYMBOL_TTL", "1800")), search_ttl=int(os.getenv("QUERY_CACHE_SEARCH_TTL", "600")), semantic_search_ttl=int(os.getenv("QUERY_CACHE_SEMANTIC_TTL", "3600")), ) query_cache = QueryResultCache(cache_manager, query_cache_config) logger.info("Query result cache initialized successfully") settings = get_settings() # Check for portable index first workspace_root = Path(".") discovery = IndexDiscovery(workspace_root) if discovery.is_index_enabled(): logger.info("MCP portable index detected") required_schema = settings.index_schema_version required_model = settings.semantic_embedding_model strict_compatibility = settings.strict_index_compatibility recovery_branch = os.getenv("MCP_INDEX_RECOVERY_BRANCH") recovery_commit = os.getenv("MCP_INDEX_RECOVERY_COMMIT") # Try to use existing compatible index index_path = discovery.get_local_index_path( requested_schema_version=required_schema, requested_embedding_model=required_model, strict_compatibility=strict_compatibility, ) # Optional branch/commit-targeted recovery path. if not index_path and (recovery_branch or recovery_commit): logger.info( "Attempting branch/commit index recovery (branch=%s, commit=%s)", recovery_branch, recovery_commit, ) if discovery.download_recovery_index( branch=recovery_branch, commit=recovery_commit, requested_schema_version=required_schema, requested_embedding_model=required_model, strict_compatibility=strict_compatibility, ): index_path = discovery.get_local_index_path( requested_schema_version=required_schema, requested_embedding_model=required_model, strict_compatibility=strict_compatibility, ) if not index_path and discovery.should_download_index( requested_schema_version=required_schema, requested_embedding_model=required_model, strict_compatibility=strict_compatibility, ): logger.info("Attempting to download index from GitHub artifacts...") if discovery.download_latest_index_for_runtime( requested_schema_version=required_schema, requested_embedding_model=required_model, strict_compatibility=strict_compatibility, ): index_path = discovery.get_local_index_path( requested_schema_version=required_schema, requested_embedding_model=required_model, strict_compatibility=strict_compatibility, ) logger.info("Successfully downloaded index from artifacts") else: logger.info("Could not download index, will use default") if index_path: logger.info(f"Using portable index: {index_path}") sqlite_store = SQLiteStore(str(index_path)) # Log index info info = discovery.get_index_info() if info["metadata"]: meta = info["metadata"] logger.info(f"Index created: {meta.get('created_at', 'unknown')}") logger.info(f"Index commit: {meta.get('commit', 'unknown')[:8]}") else: logger.info("No portable index found, using default") sqlite_store = SQLiteStore("code_index.db") else: # Initialize SQLite store with default logger.info("Initializing SQLite store with default path...") sqlite_store = SQLiteStore("code_index.db") # Initialize RepoResolver for per-request repo context resolution. _local_repo_registry = None try: from .storage.repository_registry import RepositoryRegistry _local_repo_registry = RepositoryRegistry() _store_registry = StoreRegistry.for_registry(_local_repo_registry) repo_resolver = RepoResolver(_local_repo_registry, _store_registry) import mcp_server.gateway as _gw_self _gw_self._repo_registry = _local_repo_registry logger.info("RepoResolver initialized") except Exception as _e: logger.warning("RepoResolver init failed; falling back to global sqlite_store: %s", _e) repo_resolver = None profile_registry = None requested_profiles: Dict[str, Optional[str]] = {} try: profile_registry = SemanticProfileRegistry.from_raw( settings.get_semantic_profiles_config(), settings.get_semantic_default_profile(), tool_version=settings.app_version, ) requested_profiles = { profile_id: profile.compatibility_fingerprint for profile_id, profile in profile_registry.list().items() } except Exception as exc: logger.warning("Failed to load semantic profile registry for hydration: %s", exc) selected_index_path = ( discovery.get_local_index_path() if discovery.is_index_enabled() else None ) profile_hydration_status = discovery.get_profile_hydration_status( requested_profiles=requested_profiles, index_path=selected_index_path, lexical_available=sqlite_store is not None, branch=os.getenv("MCP_INDEX_RECOVERY_BRANCH"), commit=os.getenv("MCP_INDEX_RECOVERY_COMMIT"), ) logger.info( "Profile hydration fallback strategy: %s", profile_hydration_status.get("fallback_strategy", "lexical_only"), ) logger.info("SQLite store initialized successfully") # Initialize plugin system with dynamic discovery logger.info("Initializing plugin system with dynamic discovery...") from .plugin_system.discovery import get_plugin_discovery from .plugin_system.loader import get_plugin_loader # Discover all available plugins plugin_discovery = get_plugin_discovery() discovered = plugin_discovery.discover_plugins() logger.info(f"Discovered {len(discovered)} plugins: {list(discovered.keys())}") # Initialize plugin loader plugin_loader = get_plugin_loader() # Load plugins based on configuration or auto-detected repo languages language_detection_status = None config_path = Path("plugins.yaml") enabled_languages = list(discovered.keys()) if settings.mcp_auto_detect_languages: detection = detect_repository_languages( repo_path=Path("."), max_files=settings.mcp_language_detect_max_files, min_files=settings.mcp_language_detect_min_files, ) language_detection_status = detection.to_dict() detected = set(detection.detected_languages) discovered_set = set(discovered.keys()) alias_map = { "c_sharp": "csharp", "js": "javascript", } normalized_detected = {alias_map.get(lang, lang) for lang in detected} auto_enabled = sorted(normalized_detected.intersection(discovered_set)) for helper_lang in ["plaintext", "markdown"]: if helper_lang in discovered_set and helper_lang not in auto_enabled: auto_enabled.append(helper_lang) if auto_enabled: enabled_languages = auto_enabled logger.info( "Auto-detected plugin allowlist (%d/%d discovered): %s", len(enabled_languages), len(discovered_set), enabled_languages, ) else: logger.warning( "Language detection found no matching plugin keys; falling back to all discovered plugins" ) elif config_path.exists(): import yaml with open(config_path, "r") as f: plugin_config = yaml.safe_load(f) enabled_languages = plugin_config.get("enabled_languages", list(discovered.keys())) logger.info( "Auto-detect disabled; loading plugins from plugins.yaml: %s", enabled_languages, ) else: logger.info( "Auto-detect disabled and no plugins.yaml found; loading all discovered plugins" ) # Load plugins plugin_instances = [] for language in enabled_languages: try: plugin = plugin_loader.load_plugin(language) if plugin: plugin_instances.append(plugin) logger.info(f"Successfully loaded plugin for {language}") except Exception as e: logger.error(f"Failed to load plugin for {language}: {e}") logger.info(f"Loaded {len(plugin_instances)} plugins") # Create plugin manager for compatibility plugin_manager = PluginManager(sqlite_store=sqlite_store) # Register loaded plugins with manager for plugin in plugin_instances: if hasattr(plugin, "get_language"): lang = plugin.get_language() plugin_manager._plugins[lang] = plugin logger.info(f"Loaded {len(plugin_instances)} active plugins") semantic_indexer_registry = None if settings.semantic_search_enabled and _local_repo_registry is not None: try: from .utils.semantic_indexer_registry import SemanticIndexerRegistry semantic_indexer_registry = SemanticIndexerRegistry(_local_repo_registry) except Exception as _sem_reg_err: logger.warning("Semantic registry unavailable: %s", _sem_reg_err) # Create a new EnhancedDispatcher instance with the loaded plugins logger.info("Creating dispatcher...") dispatcher = EnhancedDispatcher( semantic_search_enabled=settings.semantic_search_enabled, lazy_load=settings.mcp_fast_startup, semantic_indexer_registry=semantic_indexer_registry, ) logger.info( "EnhancedDispatcher created with semantic search enabled: %s", settings.semantic_search_enabled, ) # Initialize BM25 indexer logger.info("Initializing BM25 indexer...") bm25_indexer = BM25Indexer(sqlite_store) logger.info("BM25 indexer initialized successfully") # Initialize Fuzzy indexer logger.info("Initializing Fuzzy indexer...") fuzzy_indexer = FuzzyIndexer(sqlite_store) logger.info("Fuzzy indexer initialized successfully") semantic_runtime_enabled = settings.semantic_search_enabled semantic_setup_report = run_semantic_preflight( settings=settings, profile=settings.get_semantic_default_profile(), strict=settings.semantic_strict_mode, ) if ( semantic_runtime_enabled and settings.semantic_autostart_qdrant and not semantic_setup_report.qdrant.ok ): autostart_result = ensure_qdrant_running(settings) logger.info("Qdrant autostart result: %s", autostart_result.message) semantic_setup_report = run_semantic_preflight( settings=settings, profile=settings.get_semantic_default_profile(), strict=settings.semantic_strict_mode, ) semantic_setup_status = semantic_setup_report.to_dict() if semantic_setup_report.overall_ready: logger.info("Semantic preflight passed") else: logger.warning("Semantic preflight warnings: %s", semantic_setup_report.warnings) if settings.semantic_strict_mode and not semantic_setup_report.overall_ready: raise RuntimeError( "Semantic strict mode is enabled and preflight failed: " + "; ".join(semantic_setup_report.warnings) ) if not semantic_setup_report.overall_ready: semantic_runtime_enabled = False # Check if semantic indexer is available and enabled semantic_indexer = None if semantic_runtime_enabled: try: from .utils.semantic_indexer import SemanticIndexer # Use the canonical local file-backed semantic baseline by default. qdrant_path = os.getenv("QDRANT_PATH", "vector_index.qdrant") semantic_profile_id = settings.get_semantic_default_profile() semantic_profile = profile_registry.get(semantic_profile_id) if semantic_profile is None: raise RuntimeError( f"Semantic default profile '{semantic_profile_id}' is not available" ) semantic_indexer = SemanticIndexer( collection=_get_profile_collection_name( semantic_profile, settings.semantic_collection_name ), qdrant_path=qdrant_path, profile_registry=profile_registry, semantic_profile=semantic_profile_id, ) logger.info( "Semantic indexer initialized successfully with Qdrant at %s " "(profile=%s, collection=%s)", qdrant_path, semantic_profile_id, semantic_indexer.collection, ) except ImportError: logger.warning("Semantic indexer not available (missing dependencies)") except Exception as e: logger.error(f"Failed to initialize semantic indexer: {e}") else: logger.info( "Semantic indexer disabled after preflight; lexical/bm25/fuzzy remain active" ) # Initialize Hybrid Search logger.info("Initializing Hybrid Search...") hybrid_config = HybridSearchConfig( bm25_weight=float(os.getenv("HYBRID_BM25_WEIGHT", "0.5")), semantic_weight=float(os.getenv("HYBRID_SEMANTIC_WEIGHT", "0.3")), fuzzy_weight=float(os.getenv("HYBRID_FUZZY_WEIGHT", "0.2")), enable_bm25=True, enable_semantic=semantic_indexer is not None, enable_fuzzy=True, parallel_execution=True, cache_results=True, ) hybrid_search = HybridSearch( storage=sqlite_store, bm25_indexer=bm25_indexer, semantic_indexer=semantic_indexer, fuzzy_indexer=fuzzy_indexer, config=hybrid_config, ) logger.info( f"Hybrid Search initialized (BM25: {hybrid_config.enable_bm25}, Semantic: {hybrid_config.enable_semantic}, Fuzzy: {hybrid_config.enable_fuzzy})" ) from .storage.git_index_manager import GitAwareIndexManager git_index_manager = GitAwareIndexManager( registry=_repo_registry, dispatcher=dispatcher, repo_resolver=repo_resolver, store_registry=_store_registry, ) # Initialize multi-repo watcher + ref poller if settings.mcp_fast_startup: logger.info("MCP fast startup enabled: skipping watcher start") multi_watcher = None ref_poller = None else: logger.info("Starting MultiRepositoryWatcher and RefPoller...") try: multi_watcher = MultiRepositoryWatcher( registry=_repo_registry, dispatcher=dispatcher, index_manager=git_index_manager, repo_resolver=repo_resolver, store_registry=_store_registry, semantic_indexer_registry=semantic_indexer_registry, plugin_set_registry=getattr(dispatcher, "_plugin_set_registry", None), ) ref_poller = RefPoller( registry=_repo_registry, git_index_manager=git_index_manager, dispatcher=dispatcher, repo_resolver=repo_resolver, ) multi_watcher.start_watching_all() ref_poller.start() logger.info("MultiRepositoryWatcher and RefPoller started") except Exception as _watcher_err: logger.warning("MultiRepositoryWatcher failed to start: %s", _watcher_err) multi_watcher = None ref_poller = None # Store in app.state for potential future use app.state.dispatcher = dispatcher app.state.sqlite_store = sqlite_store app.state.file_watcher = multi_watcher app.state.plugin_manager = plugin_manager app.state.auth_manager = auth_manager app.state.security_config = security_config app.state.cache_manager = cache_manager app.state.query_cache = query_cache app.state.metrics_collector = metrics_collector app.state.health_checker = health_checker app.state.business_metrics = business_metrics app.state.bm25_indexer = bm25_indexer app.state.hybrid_search = hybrid_search app.state.fuzzy_indexer = fuzzy_indexer app.state.git_index_manager = git_index_manager app.state.semantic_indexer = semantic_indexer app.state.profile_hydration = profile_hydration_status app.state.semantic_setup = semantic_setup_status app.state.language_detection = language_detection_status app.state.preflight = { "status": preflight_result.status, "checks": [check.__dict__ for check in preflight_result.checks], } # Update status to include search capabilities search_capabilities = [] if bm25_indexer: search_capabilities.append("bm25") if fuzzy_indexer: search_capabilities.append("fuzzy") if semantic_indexer: search_capabilities.append("semantic") if hybrid_search: search_capabilities.append("hybrid") logger.info(f"Search capabilities: {', '.join(search_capabilities)}") # Register health checks for system components logger.info("Registering component health checks...") health_checker.register_health_check( "database", health_checker.create_database_health_check("code_index.db") ) health_checker.register_health_check( "plugins", health_checker.create_plugin_health_check(plugin_manager) ) # Update system metrics business_metrics.update_system_metrics( active_plugins=len(plugin_instances), indexed_files=0, # Will be updated as files are indexed database_size=0, # Will be updated periodically memory_usage=0, # Will be updated by middleware ) # Log loaded plugins with detailed status plugin_status = plugin_manager.get_detailed_plugin_status() for name, status in plugin_status.items(): basic_info = status["basic_info"] runtime_info = status["runtime_info"] logger.info( f"Plugin '{name}': {runtime_info['state']} (v{basic_info['version']}, language: {basic_info['language']}, enabled: {runtime_info['enabled']})" ) if runtime_info.get("error"): logger.warning(f"Plugin '{name}' has error: {runtime_info['error']}") logger.info( "MCP Server initialized successfully with dynamic plugin system, SQLite persistence, and file watcher" ) except Exception as e: logger.error(f"Failed to initialize MCP Server: {e}", exc_info=True) raise @app.on_event("shutdown") async def shutdown_event(): """Clean up resources on shutdown.""" # These globals are only read, not assigned, so no 'global' declaration needed if multi_watcher: try: multi_watcher.stop_watching_all() logger.info("MultiRepositoryWatcher stopped successfully") except Exception as e: logger.error(f"Error stopping MultiRepositoryWatcher: {e}", exc_info=True) if ref_poller: try: ref_poller.stop() logger.info("RefPoller stopped successfully") except Exception as e: logger.error(f"Error stopping RefPoller: {e}", exc_info=True) if dispatcher: try: dispatcher.shutdown() logger.info("Dispatcher plugin workers stopped successfully") except Exception as e: logger.error(f"Error stopping dispatcher plugin workers: {e}", exc_info=True) if plugin_manager: try: shutdown_result = plugin_manager.shutdown_safe() if shutdown_result.success: logger.info("Plugin manager shutdown successfully") else: logger.error(f"Plugin manager shutdown failed: {shutdown_result.error.message}") logger.error(f"Shutdown error details: {shutdown_result.error.details}") except Exception as e: logger.error(f"Error shutting down plugin manager: {e}", exc_info=True) if cache_manager: try: await cache_manager.shutdown() logger.info("Cache manager shutdown successfully") except Exception as e: logger.error(f"Error shutting down cache manager: {e}", exc_info=True) # Authentication endpoints @app.post("/api/v1/auth/login") async def login(credentials: AuthCredentials) -> Dict[str, Any]: """User login endpoint.""" if auth_manager is None: raise HTTPException(503, "Authentication service not ready") try: user = await auth_manager.authenticate_user(credentials) if not user: raise HTTPException(401, "Invalid credentials") access_token = await auth_manager.create_access_token(user) refresh_token = await auth_manager.create_refresh_token(user) return { "access_token": access_token, "refresh_token": refresh_token, "token_type": "bearer", "expires_in": security_config.access_token_expire_minutes * 60, "user": { "id": user.id, "username": user.username, "role": user.role.value, "permissions": [p.value for p in user.permissions], }, } except Exception as e: logger.error(f"Login failed for user '{credentials.username}': {e}") raise HTTPException(401, "Authentication failed") @app.post("/api/v1/auth/refresh") async def refresh_token(refresh_token: str) -> Dict[str, Any]: """Refresh access token.""" if auth_manager is None: raise HTTPException(503, "Authentication service not ready") try: new_access_token = await auth_manager.refresh_access_token(refresh_token) if not new_access_token: raise HTTPException(401, "Invalid refresh token") return { "access_token": new_access_token, "token_type": "bearer", "expires_in": security_config.access_token_expire_minutes * 60, } except Exception as e: logger.error(f"Token refresh failed: {e}") raise HTTPException(401, "Token refresh failed") @app.post("/api/v1/auth/logout") async def logout( refresh_token: Optional[str] = None, current_user: User = Depends(get_current_active_user), ) -> Dict[str, str]: """User logout endpoint.""" if auth_manager is None: raise HTTPException(503, "Authentication service not ready") try: if refresh_token: await auth_manager.revoke_refresh_token(refresh_token) await auth_manager._log_security_event( "user_logout", user_id=current_user.id, username=current_user.username ) return {"message": "Successfully logged out"} except Exception as e: logger.error(f"Logout failed: {e}") raise HTTPException(500, "Logout failed") @app.get("/api/v1/auth/me") async def get_current_user_info( current_user: User = Depends(get_current_active_user), ) -> Dict[str, Any]: """Get current user information.""" return { "id": current_user.id, "username": current_user.username, "email": current_user.email, "role": current_user.role.value, "permissions": [p.value for p in current_user.permissions], "is_active": current_user.is_active, "created_at": current_user.created_at.isoformat(), "last_login": (current_user.last_login.isoformat() if current_user.last_login else None), } @app.post("/api/v1/auth/register") async def register( credentials: AuthCredentials, email: Optional[str] = None, admin_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Register new user (admin only).""" if auth_manager is None: raise HTTPException(503, "Authentication service not ready") try: user = await auth_manager.create_user( username=credentials.username, password=credentials.password, email=email, role=UserRole.USER, ) return { "message": "User created successfully", "user": { "id": user.id, "username": user.username, "email": user.email, "role": user.role.value, }, } except Exception as e: logger.error(f"User registration failed: {e}") raise HTTPException(400, str(e)) # Security management endpoints @app.get("/api/v1/security/events") async def get_security_events( limit: int = 100, admin_user: User = Depends(require_role(UserRole.ADMIN)) ) -> Dict[str, Any]: """Get security events (admin only).""" if auth_manager is None: raise HTTPException(503, "Authentication service not ready") try: events = await auth_manager.get_security_events(limit) return { "events": [ { "id": event.id, "event_type": event.event_type, "user_id": event.user_id, "username": event.username, "ip_address": event.ip_address, "timestamp": event.timestamp.isoformat(), "details": event.details, "severity": event.severity, } for event in events ] } except Exception as e: logger.error(f"Failed to get security events: {e}") raise HTTPException(500, "Failed to retrieve security events") # Health check endpoints (public) @app.get("/health") async def health_check() -> Dict[str, Any]: """Basic health check endpoint.""" return {"status": "healthy", "service": "mcp-server", "timestamp": time.time()} @app.get("/health/detailed") async def detailed_health_check() -> Dict[str, Any]: """Detailed health check endpoint.""" try: overall_health = await health_checker.get_overall_health() component_results = await health_checker.check_all_components() return { "status": overall_health.status.value, "message": overall_health.message, "timestamp": overall_health.timestamp, "details": overall_health.details, "components": [ { "component": result.component, "status": result.status.value, "message": result.message, "details": result.details, } for result in component_results ], } except Exception as e: logger.error(f"Health check failed: {e}", exc_info=True) return { "status": "unhealthy", "message": f"Health check failed: {str(e)}", "timestamp": time.time(), } @app.get("/health/{component}") async def component_health_check(component: str) -> Dict[str, Any]: """Health check for a specific component.""" try: result = await health_checker.check_component(component) return { "component": result.component, "status": result.status.value, "message": result.message, "timestamp": result.timestamp, "details": result.details, } except Exception as e: logger.error(f"Component health check failed for {component}: {e}", exc_info=True) raise HTTPException(500, f"Health check failed: {str(e)}") @app.get("/ready") async def readiness_probe() -> Dict[str, Any]: """Readiness probe: 200 when dispatcher, sqlite_store, and _repo_registry are all initialised.""" from fastapi.responses import JSONResponse as _JSONResponse from .health.probes import HealthView hv = HealthView( dispatcher=dispatcher, sqlite_store=sqlite_store, registry=_repo_registry, startup_time=getattr(app.state, "startup_time", None), ) snap = hv.snapshot() code = 200 if (snap["sqlite"] and snap["registry"] and snap["dispatcher"]) else 503 return _JSONResponse(content=snap, status_code=code) @app.get("/liveness") async def liveness_probe() -> Dict[str, Any]: """Liveness probe: returns 200 if the asyncio event loop is responsive.""" import asyncio from fastapi.responses import JSONResponse as _JSONResponse await asyncio.sleep(0) return _JSONResponse(content={"status": "ok"}, status_code=200) # Metrics endpoints @app.get("/metrics", dependencies=[Depends(require_auth("metrics"))]) def get_prometheus_metrics() -> Response: """Prometheus metrics endpoint.""" try: prometheus_exporter = get_prometheus_exporter() # Update build info prometheus_exporter.set_build_info( version="1.0.0", commit=os.getenv("GIT_COMMIT", "unknown"), build_time=os.getenv("BUILD_TIME", "unknown"), ) # Update system metrics import psutil process = psutil.Process() memory_info = process.memory_info() prometheus_exporter.set_memory_usage(memory_info.rss, memory_info.vms) prometheus_exporter.set_cpu_usage(process.cpu_percent()) prometheus_exporter.set_active_threads(threading.active_count()) # Update plugin metrics if plugin_loader: _ = plugin_loader.get_statistics() for lang, plugin in plugin_loader.get_active_plugins().items(): prometheus_exporter.plugin_status.labels( plugin=plugin.__class__.__name__, language=lang ).set(1) # Update watcher metrics (placeholder — MultiRepositoryWatcher has no get_watched_count yet) if multi_watcher: pass # Generate metrics metrics = prometheus_exporter.generate_metrics() return Response(content=metrics, media_type=prometheus_exporter.get_content_type()) except Exception as e: logger.error(f"Failed to generate Prometheus metrics: {e}", exc_info=True) raise HTTPException(500, f"Failed to generate metrics: {str(e)}") @app.get("/metrics/json") def get_metrics_json( current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """JSON metrics endpoint for programmatic access.""" try: families = metrics_collector.get_metric_families() stats = metrics_collector.get_stats() return { "timestamp": time.time(), "collector_stats": stats, "metric_families": families, } except Exception as e: logger.error(f"Failed to get JSON metrics: {e}", exc_info=True) raise HTTPException(500, f"Failed to get metrics: {str(e)}") @app.get("/symbol", response_model=SymbolDef | None) async def symbol( request: Request, symbol: str, current_user: User = Depends(require_permission(Permission.READ)) ): if dispatcher is None: logger.error("Symbol lookup attempted but dispatcher not ready") raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) start_time = time.time() try: logger.debug(f"Looking up symbol: {symbol} for user: {current_user.username}") # Try cache first if query cache is available cached_result = None if query_cache and query_cache.config.enabled: cached_result = await query_cache.get_cached_result( QueryType.SYMBOL_LOOKUP, symbol=symbol, repo_id=ctx.repo_id ) if cached_result is not None: logger.debug(f"Found cached symbol: {symbol}") duration = time.time() - start_time business_metrics.record_search_performed( query=symbol, semantic=False, results_count=1, duration=duration ) return cached_result # Record symbol lookup metrics with metrics_collector.time_function("symbol_lookup"): result = dispatcher.lookup(ctx, symbol) # Cache the result if available if query_cache and query_cache.config.enabled and result: await query_cache.cache_result( QueryType.SYMBOL_LOOKUP, result, symbol=symbol, repo_id=ctx.repo_id ) # Record business metrics duration = time.time() - start_time business_metrics.record_search_performed( query=symbol, semantic=False, results_count=1 if result else 0, duration=duration, ) if result: logger.debug(f"Found symbol: {symbol}") else: logger.debug(f"Symbol not found: {symbol}") return result except Exception as e: duration = time.time() - start_time business_metrics.record_search_performed( query=symbol, semantic=False, results_count=0, duration=duration ) logger.error(f"Error looking up symbol '{symbol}': {e}", exc_info=True) raise HTTPException(500, f"Internal error during symbol lookup: {str(e)}") @app.get("/search", response_model=list[dict[str, Any]]) async def search( request: Request, q: str, semantic: bool = False, limit: int = 20, mode: str = "auto", # "auto", "hybrid", "bm25", "semantic", "fuzzy", "classic" language: Optional[str] = None, file_filter: Optional[str] = None, source_type: Optional[str] = None, friction_categories: Optional[str] = None, history_labels: Optional[str] = None, history_repos: Optional[str] = None, include_source_metadata: bool = False, current_user: User = Depends(require_permission(Permission.READ)), ): """Search with support for multiple modes including hybrid search. Args: q: Search query semantic: Whether to use semantic search (for backward compatibility) limit: Maximum number of results mode: Search mode - "auto" (default), "hybrid", "bm25", "semantic", "fuzzy", or "classic" language: Filter by programming language file_filter: Filter by file path pattern current_user: Authenticated user """ if dispatcher is None and mode == "classic": logger.error("Search attempted but dispatcher not ready") raise HTTPException(503, "Dispatcher not ready") try: options = build_search_options( query=q, repository=request.query_params.get("repository"), semantic=semantic, limit=limit, source_type=source_type, friction_categories=friction_categories, history_labels=history_labels, history_repos=history_repos, include_source_metadata=include_source_metadata, ) except ClientValidationError as exc: detail: dict[str, Any] = {"error": exc.error, "details": exc.details} if exc.allowed_categories: detail["allowed_categories"] = exc.allowed_categories raise HTTPException(400, detail) from exc except ValueError as exc: raise HTTPException( 400, {"error": "Invalid search source filters", "details": str(exc)} ) from exc ctx = get_repo_ctx(request) repo_bm25: Optional[BM25Indexer] = None repo_fuzzy: Optional[FuzzyIndexer] = None repo_hybrid: Optional[HybridSearch] = None if mode in {"hybrid", "bm25", "fuzzy"} or ctx.repo_id == _FALLBACK_REPO_ID: repo_bm25, repo_fuzzy, repo_hybrid = _search_backends_for_repo(ctx) explicit_repository = request.query_params.get("repository") service_repo_resolver = repo_resolver if explicit_repository is None and ctx.repo_id == _FALLBACK_REPO_ID: service_repo_resolver = None start_time = time.time() try: # Determine effective search mode effective_mode = mode filtered_search_requested = bool( options.source_type or options.friction_categories or options.history_labels or options.history_repos or options.include_source_metadata ) if mode == "auto": if filtered_search_requested and dispatcher and hasattr(dispatcher, "search"): effective_mode = "semantic" if semantic else "classic" elif semantic and dispatcher and hasattr(dispatcher, "search"): effective_mode = "semantic" elif ctx.repo_id != _FALLBACK_REPO_ID: effective_mode = "classic" # Non-semantic requests use hybrid if available, otherwise fall back. elif repo_hybrid is not None: effective_mode = "hybrid" else: effective_mode = "bm25" if repo_bm25 else "classic" logger.debug( f"Searching for: '{q}' (mode={effective_mode}, limit={limit}, language={language}) for user: {current_user.username}" ) # Build filters filters = {} if language: filters["language"] = language if file_filter: filters["file_filter"] = file_filter cache_params = { "repo_id": ctx.repo_id, "q": q, "semantic": effective_mode == "semantic", "limit": limit, "language": language, "file_filter": file_filter, "source_type": options.source_type.value if options.source_type else None, "friction_categories": options.friction_categories, "history_labels": options.history_labels, "history_repos": options.history_repos, "include_source_metadata": options.include_source_metadata, } cached_results = None if query_cache and query_cache.config.enabled: query_type = ( QueryType.SEMANTIC_SEARCH if effective_mode == "semantic" else QueryType.SEARCH ) cached_results = await query_cache.get_cached_result(query_type, **cache_params) if cached_results is not None: cached_results = [ r for r in (_normalize_search_result(x) for x in cached_results) if r is not None ] logger.debug(f"Found cached search results for: '{q}' ({len(cached_results)} results)") duration = time.time() - start_time business_metrics.record_search_performed( query=q, semantic=(effective_mode == "semantic"), results_count=len(cached_results), duration=duration, ) return cached_results # Perform search based on mode results = [] if effective_mode == "hybrid" and repo_hybrid: # Use hybrid search with metrics_collector.time_function("search", labels={"mode": "hybrid"}): hybrid_results = await repo_hybrid.search(query=q, filters=filters, limit=limit) results = [ r for r in (_normalize_search_result(x) for x in hybrid_results) if r is not None ] elif effective_mode == "bm25" and repo_bm25: # Direct BM25 search with metrics_collector.time_function("search", labels={"mode": "bm25"}): bm25_results = repo_bm25.search(q, limit=limit, **filters) if not bm25_results: bm25_results = ctx.sqlite_store.search_bm25(q, table="fts_code", limit=limit) results = [ r for r in (_normalize_search_result(x) for x in bm25_results) if r is not None ] elif effective_mode == "fuzzy" and repo_fuzzy: # Direct fuzzy search with metrics_collector.time_function("search", labels={"mode": "fuzzy"}): if hasattr(repo_fuzzy, "search_fuzzy"): fuzzy_results = repo_fuzzy.search_fuzzy(q, max_results=limit) else: fuzzy_results = repo_fuzzy.search(q, limit=limit) results = [ r for r in (_normalize_search_result(x) for x in fuzzy_results) if r is not None ] elif effective_mode == "semantic": # Use classic dispatcher with semantic=True if dispatcher: with metrics_collector.time_function("search", labels={"mode": "semantic"}): result = execute_search_service( dispatcher=dispatcher, repo_resolver=service_repo_resolver, options=options, workspace_root=ctx.workspace_root, ctx=ctx, ) if result.index_unavailable is not None: raise HTTPException( 503, detail={ "error": "Index unavailable", "code": result.index_unavailable.code, "safe_fallback": result.index_unavailable.safe_fallback, "readiness": result.index_unavailable.readiness, "message": result.index_unavailable.message, "remediation": result.index_unavailable.remediation, }, ) if result.code == "semantic_not_ready": raise HTTPException( 503, detail={ "error": "Semantic search not ready", "code": result.code, "semantic_readiness": result.semantic_readiness, "message": result.message, }, ) if result.code == "semantic_search_failed": raise HTTPException( 500, detail={ "error": "Semantic search failed", "code": result.code, "semantic_readiness": result.semantic_readiness, "message": result.message, }, ) results = search_result_for_gateway(result) else: raise HTTPException( 503, detail={ "error": "Semantic search not available", "reason": "Semantic preflight failed (embedding endpoint or credentials unavailable)", "setup": { "recommended": "Run: python scripts/cli/mcp_cli.py setup semantic", "method_1_mcp_json": [ "Configure in .mcp.json (recommended for Claude Code):", "{", ' "mcpServers": {', ' "code-index-mcp": {', ' "command": "uvicorn",', ' "args": ["mcp_server.gateway:app"],', ' "env": {', ' "VOYAGE_API_KEY": "your-key-here",', ' "OPENAI_API_BASE": "http://localhost:8001/v1",', ' "SEMANTIC_SEARCH_ENABLED": "true"', " }", " }", " }", "}", ], "method_2_cli": [ "Or use Claude Code CLI:", "claude mcp add code-index-mcp -e VOYAGE_API_KEY=your_key -e SEMANTIC_SEARCH_ENABLED=true -- uvicorn mcp_server.gateway:app", ], "method_3_env": [ "Or set environment variables:", "export VOYAGE_API_KEY=your_key", "export SEMANTIC_SEARCH_ENABLED=true", ], "method_4_dotenv": [ "Or add to .env file:", "VOYAGE_API_KEY=your_key", "SEMANTIC_SEARCH_ENABLED=true", ], "get_api_key": "Get your API key from: https://www.voyageai.com/", "alternative": "Use mode='hybrid' or mode='bm25' for keyword-based search", }, }, ) else: # Classic search through dispatcher if dispatcher: with metrics_collector.time_function("search", labels={"mode": "classic"}): result = execute_search_service( dispatcher=dispatcher, repo_resolver=service_repo_resolver, options=options, workspace_root=ctx.workspace_root, ctx=ctx, ) if result.index_unavailable is not None: raise HTTPException( 503, detail={ "error": "Index unavailable", "code": result.index_unavailable.code, "safe_fallback": result.index_unavailable.safe_fallback, "readiness": result.index_unavailable.readiness, "message": result.index_unavailable.message, "remediation": result.index_unavailable.remediation, }, ) results = search_result_for_gateway(result) else: raise HTTPException(503, "Classic search not available") # Cache the results if available results = [r for r in (_normalize_search_result(x) for x in results) if r is not None] if query_cache and query_cache.config.enabled and results: query_type = ( QueryType.SEMANTIC_SEARCH if effective_mode == "semantic" else QueryType.SEARCH ) await query_cache.cache_result( query_type, results, **cache_params, ) # Record business metrics duration = time.time() - start_time business_metrics.record_search_performed( query=q, semantic=(effective_mode == "semantic"), results_count=len(results), duration=duration, ) logger.debug(f"Search returned {len(results)} results using {effective_mode} mode") return results except HTTPException: raise except Exception as e: duration = time.time() - start_time business_metrics.record_search_performed( query=q, semantic=semantic, results_count=0, duration=duration ) logger.error(f"Error during search for '{q}': {e}", exc_info=True) raise HTTPException(500, f"Internal error during search: {str(e)}") @app.get("/search/capabilities") async def get_search_capabilities( current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get available search capabilities and configuration guidance.""" voyage_key = os.environ.get("VOYAGE_API_KEY") openai_base = os.environ.get("OPENAI_API_BASE") semantic_enabled = os.environ.get("SEMANTIC_SEARCH_ENABLED", "false").lower() == "true" return { "available_modes": { "bm25": bm25_indexer is not None, "fuzzy": fuzzy_indexer is not None, "semantic": semantic_indexer is not None, "hybrid": hybrid_search is not None, "classic": dispatcher is not None, }, "semantic_config": { "enabled": semantic_indexer is not None, "api_key_configured": bool(voyage_key), "openai_api_base": openai_base, "semantic_enabled_flag": semantic_enabled, "status": "ready" if semantic_indexer else "not_configured", "setup": semantic_setup_status, }, "language_detection": language_detection_status, "configuration_guide": { "mcp_json_example": { "description": "Add to .mcp.json for Claude Code (recommended)", "config": { "mcpServers": { "code-index-mcp": { "command": "uvicorn", "args": ["mcp_server.gateway:app"], "env": { "VOYAGE_API_KEY": "your-key-here", "OPENAI_API_BASE": "http://localhost:8001/v1", "SEMANTIC_SEARCH_ENABLED": "true", }, } } }, }, "cli_command": "claude mcp add code-index-mcp -e VOYAGE_API_KEY=key -e SEMANTIC_SEARCH_ENABLED=true -- uvicorn mcp_server.gateway:app", "env_file": "Add to .env: VOYAGE_API_KEY=key and SEMANTIC_SEARCH_ENABLED=true", "get_api_key": "https://www.voyageai.com/", }, } @app.get("/status") async def get_status( request: Request, current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Returns server status including plugin information and statistics.""" if dispatcher is None: return { "status": "error", "plugins": 0, "indexed_files": {"total": 0, "by_language": {}}, "profile_hydration": profile_hydration_status, "version": "0.1.0", "message": "Dispatcher not initialized", } explicit_selector = request.headers.get("X-Repo-Id") or request.query_params.get("repository") try: ctx: Optional[RepoContext] = get_repo_ctx(request) except HTTPException: if explicit_selector is not None: raise ctx = None cache_repo_id = ctx.repo_id if ctx is not None else "aggregate" try: # Try cache first if query cache is available cached_status = None if query_cache and query_cache.config.enabled: cached_status = await query_cache.get_cached_result( QueryType.PROJECT_STATUS, repo_id=cache_repo_id ) if cached_status is not None: return cached_status # Get plugin count via public Protocol method loaded_plugins = dispatcher.plugins() plugin_count = len(loaded_plugins) # Get indexed files statistics indexed_stats = {"total": 0, "by_language": {}} if ctx is not None and hasattr(dispatcher, "get_statistics"): indexed_stats = dispatcher.get_statistics(ctx) else: # Calculate basic statistics from plugins for plugin in loaded_plugins: if hasattr(plugin, "get_indexed_count"): count = plugin.get_indexed_count() indexed_stats["total"] += count lang = getattr(plugin, "language", getattr(plugin, "lang", "unknown")) indexed_stats["by_language"][lang] = count # Add database statistics if available db_stats = {} if ctx is not None and ctx.sqlite_store: db_stats = ctx.sqlite_store.get_statistics() # Add cache statistics if available cache_stats = {} if cache_manager: try: cache_metrics = await cache_manager.get_metrics() cache_stats = { "hit_rate": cache_metrics.hit_rate, "entries": cache_metrics.entries_count, "memory_usage_mb": cache_metrics.memory_usage_mb, } except Exception as e: logger.warning(f"Failed to get cache stats: {e}") from mcp_server.health.repo_status import build_health_row as _build_health_row _repositories = [] if _repo_registry is not None: try: _repositories = [] for info in _repo_registry.get_all_repositories().values(): _features = None if repo_resolver is not None and hasattr( dispatcher, "get_runtime_feature_status" ): try: _readiness = repo_resolver.classify(info.path) if isinstance(_readiness, RepositoryReadiness) and _readiness.ready: _ctx = ( repo_resolver.resolve_ready(info.path) if isinstance(repo_resolver, RepoResolver) else repo_resolver.resolve(info.path) ) if _ctx is not None: _features = dispatcher.get_runtime_feature_status(_ctx) except Exception: _features = None _repositories.append(_build_health_row(info, features=_features)) except Exception as _repo_err: logger.warning("Failed to build repository health rows: %s", _repo_err) status_data = { "status": "operational", "plugins": plugin_count, "indexed_files": indexed_stats, "database": db_stats, "cache": cache_stats, "search_capabilities": [], "profile_hydration": profile_hydration_status, "semantic_setup": semantic_setup_status, "language_detection": language_detection_status, "version": "0.1.0", "repositories": _repositories, } # Add search capabilities if bm25_indexer: status_data["search_capabilities"].append("bm25") if fuzzy_indexer: status_data["search_capabilities"].append("fuzzy") if hasattr(app.state, "semantic_indexer") and app.state.semantic_indexer: status_data["search_capabilities"].append("semantic") if hybrid_search: status_data["search_capabilities"].append("hybrid") # Cache the status if query_cache and query_cache.config.enabled: await query_cache.cache_result( QueryType.PROJECT_STATUS, status_data, repo_id=cache_repo_id ) return status_data except HTTPException: raise except Exception as e: logger.error(f"Error getting server status: {e}", exc_info=True) return { "status": "error", "plugins": 0, "indexed_files": {"total": 0, "by_language": {}}, "profile_hydration": profile_hydration_status, "version": "0.1.0", "message": str(e), } @app.get("/plugins") def plugins( current_user: User = Depends(require_permission(Permission.READ)), ) -> List[Dict[str, Any]]: """Returns list of loaded plugins with their information.""" if plugin_manager is None: logger.error("Plugin list requested but plugin manager not ready") raise HTTPException(503, "Plugin manager not ready") try: plugin_list = [] plugin_infos = plugin_manager._registry.list_plugins() plugin_status = plugin_manager.get_plugin_status() for info in plugin_infos: plugin_state = plugin_status.get(info.name, {}) plugin_data = { "name": info.name, "version": info.version, "description": info.description, "author": info.author, "language": info.language, "file_extensions": info.file_extensions, "state": plugin_state.get("state", "unknown"), "enabled": plugin_state.get("enabled", False), } plugin_list.append(plugin_data) logger.debug(f"Returning {len(plugin_list)} plugins") return plugin_list except Exception as e: logger.error(f"Error getting plugin list: {e}", exc_info=True) raise HTTPException(500, f"Internal error getting plugins: {str(e)}") @app.post("/reindex") async def reindex( request: Request, path: Optional[str] = None, current_user: User = Depends(require_permission(Permission.EXECUTE)), ) -> Dict[str, Any]: """Triggers manual reindexing of files. Args: path: Optional specific directory path to reindex. If not provided, reindexes all configured paths. Returns: Task status information. """ if dispatcher is None: logger.error("Reindex requested but dispatcher not ready") raise HTTPException(503, "Dispatcher not ready") selector = request.headers.get("X-Repo-Id") or request.query_params.get("repository") readiness = None if repo_resolver is not None: target = selector if selector is not None else str(Path.cwd()) try: readiness = repo_resolver.classify(target) except Exception as exc: raise HTTPException(503, detail=_repository_unavailable_detail(str(target))) from exc if readiness is not None and not readiness.ready: if path and readiness.state in _RECOVERABLE_REINDEX_STATES: raise HTTPException( 409, detail={ "error": "Full repository rebuild required", "code": "full_rebuild_required", "readiness": readiness.to_dict(), "mutation_performed": False, }, ) if not path and readiness.state in _RECOVERABLE_REINDEX_STATES: if git_index_manager is None or readiness.repository_id is None: raise HTTPException(503, detail="Staged repository reindex is unavailable") sync_result = git_index_manager.rebuild_repository_index(readiness.repository_id) if sync_result.action != "full_index": raise HTTPException( 409, detail={ "error": "Reindex failed", "code": sync_result.code or "reindex_failed", "mutation_performed": False, }, ) return { "status": "completed", "mode": "staged_full", "indexed_files": sync_result.files_processed, "commit": sync_result.commit, "readiness": sync_result.readiness, } raise HTTPException(503, detail=_repository_unavailable_detail(str(target))) ctx = get_repo_ctx(request) try: logger.info(f"Manual reindex requested for path: {path or 'all'}") if path: workspace_root = ctx.workspace_root.expanduser().resolve(strict=True) candidate = Path(path).expanduser() if not candidate.is_absolute(): candidate = workspace_root / candidate try: target_path = candidate.resolve(strict=True) except FileNotFoundError as exc: raise HTTPException(404, f"Path not found: {path}") from exc try: target_path.relative_to(workspace_root) except ValueError as exc: raise HTTPException( 403, detail={ "error": "Reindex path is outside the selected repository", "code": "path_outside_selected_repository", "repository_id": ctx.repo_id, }, ) from exc guard = _get_path_guard() if guard is not None: try: guard.normalize_and_check(target_path) except PathTraversalError as exc: raise HTTPException( 403, detail={ "error": "Reindex path is outside MCP_ALLOWED_ROOTS", "code": "path_outside_allowed_roots", }, ) from exc indexed_count = 0 if target_path.is_file(): dispatcher.index_file(ctx, target_path) indexed_count = 1 else: active_plugins = dispatcher.plugins() for file_path in target_path.rglob("*"): if file_path.is_file(): try: resolved_file = file_path.resolve(strict=True) resolved_file.relative_to(workspace_root) if guard is not None: guard.normalize_and_check(resolved_file) for plugin in active_plugins: if plugin.supports(resolved_file): dispatcher.index_file(ctx, resolved_file) indexed_count += 1 break except (OSError, ValueError, PathTraversalError) as e: logger.warning(f"Skipped unsafe reindex path {file_path}: {e}") except Exception as e: logger.warning(f"Failed to index {file_path}: {e}") logger.info(f"Successfully reindexed {indexed_count} files in {path}") return { "status": "completed", "message": f"Reindexed {indexed_count} files in {path}", } else: if ctx.repo_id == _FALLBACK_REPO_ID or git_index_manager is None: raise HTTPException( 503, detail={ "error": "Staged repository reindex is unavailable", "code": "staged_reindex_unavailable", "repository_id": ctx.repo_id, }, ) sync_result = git_index_manager.rebuild_repository_index(ctx.repo_id) if sync_result.action != "full_index": raise HTTPException( 409, detail={ "error": "Staged repository reindex was refused", "code": sync_result.code or "reindex_failed", "readiness": sync_result.readiness, "message": sync_result.error, }, ) return { "status": "completed", "mode": "staged_full", "message": f"Reindexed {sync_result.files_processed} files", "indexed_files": sync_result.files_processed, "commit": sync_result.commit, "recovery": sync_result.readiness, } except HTTPException: raise except Exception as e: logger.error(f"Reindexing failed: {e}", exc_info=True) raise HTTPException(500, f"Reindexing failed: {str(e)}") @app.post("/plugins/{plugin_name}/reload") async def reload_plugin( plugin_name: str, current_user: User = Depends(require_role(UserRole.ADMIN)) ) -> Dict[str, str]: """Reload a specific plugin. Args: plugin_name: Name of the plugin to reload Returns: Status message """ if plugin_manager is None: logger.error("Plugin reload requested but plugin manager not ready") raise HTTPException(503, "Plugin manager not ready") try: plugin_manager.reload_plugin(plugin_name) return { "status": "success", "message": f"Plugin '{plugin_name}' reloaded successfully", } except Exception as e: logger.error(f"Failed to reload plugin '{plugin_name}': {e}", exc_info=True) raise HTTPException(500, f"Failed to reload plugin: {str(e)}") @app.post("/plugins/{plugin_name}/enable") async def enable_plugin( plugin_name: str, current_user: User = Depends(require_role(UserRole.ADMIN)) ) -> Dict[str, str]: """Enable a disabled plugin. Args: plugin_name: Name of the plugin to enable Returns: Status message """ if plugin_manager is None: logger.error("Plugin enable requested but plugin manager not ready") raise HTTPException(503, "Plugin manager not ready") try: plugin_manager.enable_plugin(plugin_name) # Recreate dispatcher with updated plugins active_plugins = plugin_manager.get_active_plugins() global dispatcher dispatcher = EnhancedDispatcher(list(active_plugins.values())) return { "status": "success", "message": f"Plugin '{plugin_name}' enabled successfully", } except Exception as e: logger.error(f"Failed to enable plugin '{plugin_name}': {e}", exc_info=True) raise HTTPException(500, f"Failed to enable plugin: {str(e)}") @app.post("/plugins/{plugin_name}/disable") async def disable_plugin( plugin_name: str, current_user: User = Depends(require_role(UserRole.ADMIN)) ) -> Dict[str, str]: """Disable an active plugin. Args: plugin_name: Name of the plugin to disable Returns: Status message """ if plugin_manager is None: logger.error("Plugin disable requested but plugin manager not ready") raise HTTPException(503, "Plugin manager not ready") try: plugin_manager.disable_plugin(plugin_name) # Recreate dispatcher with updated plugins active_plugins = plugin_manager.get_active_plugins() global dispatcher dispatcher = EnhancedDispatcher(list(active_plugins.values())) return { "status": "success", "message": f"Plugin '{plugin_name}' disabled successfully", } except Exception as e: logger.error(f"Failed to disable plugin '{plugin_name}': {e}", exc_info=True) raise HTTPException(500, f"Failed to disable plugin: {str(e)}") # Cache management endpoints @app.get("/cache/stats") async def get_cache_stats( current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get cache statistics and performance metrics.""" if not cache_manager: raise HTTPException(503, "Cache manager not ready") try: cache_metrics = await cache_manager.get_metrics() backend_stats = await cache_manager.get_backend_stats() stats = { "cache_metrics": { "hits": cache_metrics.hits, "misses": cache_metrics.misses, "sets": cache_metrics.sets, "deletes": cache_metrics.deletes, "hit_rate": cache_metrics.hit_rate, "avg_response_time_ms": cache_metrics.avg_response_time_ms, "entries_count": cache_metrics.entries_count, "memory_usage_mb": cache_metrics.memory_usage_mb, }, "backend_stats": backend_stats, } # Add query cache stats if available if query_cache: query_stats = await query_cache.get_cache_stats() stats["query_cache"] = query_stats return stats except Exception as e: logger.error(f"Failed to get cache stats: {e}") raise HTTPException(500, f"Failed to get cache statistics: {str(e)}") @app.post("/cache/clear") async def clear_cache( current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Clear all cache entries (admin only).""" if not cache_manager: raise HTTPException(503, "Cache manager not ready") try: count = await cache_manager.clear() logger.info(f"Cache cleared by admin user {current_user.username}: {count} entries") return { "status": "success", "message": f"Cleared {count} cache entries", "cleared_entries": count, } except Exception as e: logger.error(f"Failed to clear cache: {e}") raise HTTPException(500, f"Failed to clear cache: {str(e)}") @app.post("/cache/invalidate") async def invalidate_cache_by_tags( tags: List[str], current_user: User = Depends(require_role(UserRole.ADMIN)) ) -> Dict[str, Any]: """Invalidate cache entries by tags (admin only).""" if not cache_manager: raise HTTPException(503, "Cache manager not ready") try: tag_set = set(tags) count = await cache_manager.invalidate_by_tags(tag_set) logger.info( f"Cache invalidated by admin user {current_user.username}: {count} entries with tags {tags}" ) return { "status": "success", "message": f"Invalidated {count} cache entries with tags {tags}", "invalidated_entries": count, "tags": tags, } except Exception as e: logger.error(f"Failed to invalidate cache by tags: {e}") raise HTTPException(500, f"Failed to invalidate cache: {str(e)}") @app.post("/cache/invalidate/files") async def invalidate_cache_by_files( file_paths: List[str], current_user: User = Depends(require_role(UserRole.ADMIN)) ) -> Dict[str, Any]: """Invalidate cache entries that depend on specific files (admin only).""" if not query_cache: raise HTTPException(503, "Query cache not ready") try: total_count = 0 for file_path in file_paths: count = await query_cache.invalidate_file_queries(file_path) total_count += count logger.info( f"File-based cache invalidation by admin user {current_user.username}: {total_count} entries for {len(file_paths)} files" ) return { "status": "success", "message": f"Invalidated {total_count} cache entries for {len(file_paths)} files", "invalidated_entries": total_count, "files": file_paths, } except Exception as e: logger.error(f"Failed to invalidate cache by files: {e}") raise HTTPException(500, f"Failed to invalidate cache by files: {str(e)}") @app.post("/cache/invalidate/semantic") async def invalidate_semantic_cache( current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Invalidate all semantic search cache entries (admin only).""" if not query_cache: raise HTTPException(503, "Query cache not ready") try: count = await query_cache.invalidate_semantic_queries() logger.info( f"Semantic cache invalidated by admin user {current_user.username}: {count} entries" ) return { "status": "success", "message": f"Invalidated {count} semantic search cache entries", "invalidated_entries": count, } except Exception as e: logger.error(f"Failed to invalidate semantic cache: {e}") raise HTTPException(500, f"Failed to invalidate semantic cache: {str(e)}") @app.post("/cache/warm") async def warm_cache( keys: List[str], current_user: User = Depends(require_role(UserRole.ADMIN)) ) -> Dict[str, Any]: """Warm cache with predefined keys (admin only).""" if not cache_manager: raise HTTPException(503, "Cache manager not ready") try: # Simple factory function for warming - would need more sophisticated logic in production async def factory(key: str): # This is a placeholder - in real implementation would depend on key type return f"warmed_value_for_{key}" count = await cache_manager.warm_cache(keys, factory) logger.info(f"Cache warmed by admin user {current_user.username}: {count} entries") return { "status": "success", "message": f"Warmed {count} cache entries", "warmed_entries": count, "requested_keys": len(keys), } except Exception as e: logger.error(f"Failed to warm cache: {e}") raise HTTPException(500, f"Failed to warm cache: {str(e)}") @app.post("/cache/cleanup") async def cleanup_cache( current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Manually trigger cache cleanup (admin only).""" if not cache_manager: raise HTTPException(503, "Cache manager not ready") try: count = await cache_manager.cleanup() logger.info( f"Cache cleanup triggered by admin user {current_user.username}: {count} entries cleaned" ) return { "status": "success", "message": f"Cleaned up {count} expired cache entries", "cleaned_entries": count, } except Exception as e: logger.error(f"Failed to cleanup cache: {e}") raise HTTPException(500, f"Failed to cleanup cache: {str(e)}") # Hybrid Search endpoints @app.get("/search/config") async def get_search_config( current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get current hybrid search configuration.""" if not hybrid_search: raise HTTPException(503, "Hybrid search not available") config = hybrid_search.config return { "weights": { "bm25": config.bm25_weight, "semantic": config.semantic_weight, "fuzzy": config.fuzzy_weight, }, "enabled_methods": { "bm25": config.enable_bm25, "semantic": config.enable_semantic, "fuzzy": config.enable_fuzzy, }, "rrf_k": config.rrf_k, "parallel_execution": config.parallel_execution, "cache_results": config.cache_results, "limits": { "individual_limit": config.individual_limit, "final_limit": config.final_limit, }, } @app.put("/search/config/weights") async def update_search_weights( bm25: Optional[float] = None, semantic: Optional[float] = None, fuzzy: Optional[float] = None, current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Update hybrid search weights (admin only). Weights will be normalized to sum to 1.0. """ if not hybrid_search: raise HTTPException(503, "Hybrid search not available") try: hybrid_search.set_weights(bm25=bm25, semantic=semantic, fuzzy=fuzzy) # Get updated config config = hybrid_search.config logger.info( f"Search weights updated by {current_user.username}: " f"BM25={config.bm25_weight:.3f}, " f"Semantic={config.semantic_weight:.3f}, " f"Fuzzy={config.fuzzy_weight:.3f}" ) return { "status": "success", "weights": { "bm25": config.bm25_weight, "semantic": config.semantic_weight, "fuzzy": config.fuzzy_weight, }, } except Exception as e: logger.error(f"Failed to update search weights: {e}") raise HTTPException(500, f"Failed to update weights: {str(e)}") @app.put("/search/config/methods") async def toggle_search_methods( bm25: Optional[bool] = None, semantic: Optional[bool] = None, fuzzy: Optional[bool] = None, current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Enable or disable search methods (admin only).""" if not hybrid_search: raise HTTPException(503, "Hybrid search not available") try: hybrid_search.enable_methods(bm25=bm25, semantic=semantic, fuzzy=fuzzy) # Get updated config config = hybrid_search.config logger.info( f"Search methods updated by {current_user.username}: " f"BM25={config.enable_bm25}, " f"Semantic={config.enable_semantic}, " f"Fuzzy={config.enable_fuzzy}" ) return { "status": "success", "enabled_methods": { "bm25": config.enable_bm25, "semantic": config.enable_semantic, "fuzzy": config.enable_fuzzy, }, } except Exception as e: logger.error(f"Failed to update search methods: {e}") raise HTTPException(500, f"Failed to update methods: {str(e)}") @app.get("/search/statistics") async def get_search_statistics( current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get search statistics and performance metrics.""" stats = {} # Hybrid search statistics if hybrid_search: stats["hybrid_search"] = hybrid_search.get_statistics() # BM25 statistics if bm25_indexer: stats["bm25"] = bm25_indexer.get_statistics() # Fuzzy search statistics if fuzzy_indexer and hasattr(fuzzy_indexer, "get_statistics"): stats["fuzzy"] = fuzzy_indexer.get_statistics() # Add general search metrics from business metrics if business_metrics: search_metrics = business_metrics.get_search_metrics() stats["general"] = { "total_searches": search_metrics.get("total_searches", 0), "average_response_time_ms": search_metrics.get("avg_response_time", 0), "search_success_rate": search_metrics.get("success_rate", 0), } return stats @app.post("/search/optimize") async def optimize_search_indexes( current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Optimize search indexes for better performance (admin only).""" results = {} try: # Optimize BM25 indexes if bm25_indexer: bm25_indexer.optimize() results["bm25"] = "optimized" logger.info("BM25 indexes optimized") # Optimize FTS5 tables in SQLite if sqlite_store: sqlite_store.optimize_fts_tables() results["fts5"] = "optimized" logger.info("FTS5 tables optimized") # Clear hybrid search cache if hybrid_search: hybrid_search.clear_cache() results["hybrid_cache"] = "cleared" logger.info("Hybrid search cache cleared") logger.info(f"Search indexes optimized by {current_user.username}") return { "status": "success", "message": "Search indexes optimized successfully", "results": results, } except Exception as e: logger.error(f"Failed to optimize search indexes: {e}") raise HTTPException(500, f"Failed to optimize indexes: {str(e)}") @app.get("/search/term/{term}/stats") async def get_term_statistics( term: str, current_user: User = Depends(require_permission(Permission.READ)) ) -> Dict[str, Any]: """Get statistics for a specific search term.""" stats = {} try: # BM25 term statistics if bm25_indexer: stats["bm25"] = bm25_indexer.get_term_statistics(term) # SQLite FTS5 statistics if sqlite_store: stats["fts5"] = sqlite_store.get_bm25_term_statistics(term) return {"term": term, "statistics": stats} except Exception as e: logger.error(f"Failed to get term statistics: {e}") raise HTTPException(500, f"Failed to get term statistics: {str(e)}") @app.post("/search/rebuild") async def rebuild_search_indexes( index_type: str = "all", # "all", "bm25", "fuzzy", "semantic" current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Rebuild search indexes (admin only).""" if index_type not in ["all", "bm25", "fuzzy", "semantic"]: raise HTTPException( 400, "Invalid index_type. Must be 'all', 'bm25', 'fuzzy', or 'semantic'" ) results = {} try: if index_type in ["all", "bm25"] and bm25_indexer: bm25_indexer.rebuild() results["bm25"] = "rebuilt" logger.info("BM25 index rebuilt") if index_type in ["all", "fuzzy"] and fuzzy_indexer: fuzzy_indexer.clear() # Re-index all files if sqlite_store: files = sqlite_store.get_all_files() for file_info in files: try: with open(file_info["path"], "r", encoding="utf-8") as f: content = f.read() fuzzy_indexer.add_file(file_info["path"], content) except Exception as e: logger.warning(f"Failed to re-index {file_info['path']}: {e}") results["fuzzy"] = "rebuilt" logger.info("Fuzzy index rebuilt") if index_type in ["all", "semantic"]: # Semantic index rebuild would go here if available if hasattr(hybrid_search, "semantic_indexer") and hybrid_search.semantic_indexer: results["semantic"] = "rebuild_not_implemented" else: results["semantic"] = "not_available" logger.info(f"Search indexes rebuilt by {current_user.username}: {index_type}") return { "status": "success", "message": "Search indexes rebuilt successfully", "index_type": index_type, "results": results, } except Exception as e: logger.error(f"Failed to rebuild search indexes: {e}") raise HTTPException(500, f"Failed to rebuild indexes: {str(e)}") # Graph Analysis Endpoints @app.get("/graph/dependencies/{symbol}") async def get_symbol_dependencies( request: Request, symbol: str, max_depth: int = 3, current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get dependencies of a symbol.""" if dispatcher is None: raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) try: dependencies = dispatcher.find_symbol_dependencies(ctx, symbol, max_depth) return { "symbol": symbol, "dependencies": dependencies, "count": len(dependencies), "max_depth": max_depth, } except Exception as e: logger.error(f"Error getting dependencies for {symbol}: {e}") raise HTTPException(500, f"Failed to get dependencies: {str(e)}") @app.get("/graph/dependents/{symbol}") async def get_symbol_dependents( request: Request, symbol: str, max_depth: int = 3, current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get dependents of a symbol (what depends on it).""" if dispatcher is None: raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) try: dependents = dispatcher.find_symbol_dependents(ctx, symbol, max_depth) return { "symbol": symbol, "dependents": dependents, "count": len(dependents), "max_depth": max_depth, } except Exception as e: logger.error(f"Error getting dependents for {symbol}: {e}") raise HTTPException(500, f"Failed to get dependents: {str(e)}") @app.get("/graph/hotspots") async def get_code_hotspots( request: Request, top_n: int = 10, current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get code hotspots (highly connected nodes).""" if dispatcher is None: raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) try: hotspots = dispatcher.get_code_hotspots(ctx, top_n) return { "hotspots": hotspots, "count": len(hotspots), "top_n": top_n, } except Exception as e: logger.error(f"Error getting hotspots: {e}") raise HTTPException(500, f"Failed to get hotspots: {str(e)}") @app.post("/graph/context") async def get_context_for_symbols( request: Request, symbols: List[str], radius: int = 2, budget: int = 200, weights: Optional[Dict[str, float]] = None, current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get optimal context for symbols using graph cut.""" if dispatcher is None: raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) try: result = dispatcher.get_context_for_symbols(ctx, symbols, radius, budget, weights) if result is None: return { "available": False, "message": "Graph features not available. Install treesitter-chunker.", } return { "available": True, "symbols": symbols, "selected_nodes": [ { "id": node.id, "symbol": node.symbol, "file": node.file_path, "kind": node.kind, "language": node.language, "line": node.line_start, "score": node.score, } for node in result.selected_nodes ], "edges": [ { "source": edge.source_id, "target": edge.target_id, "type": edge.edge_type.value, "weight": edge.weight, } for edge in result.induced_edges ], "statistics": { "selected_nodes": len(result.selected_nodes), "induced_edges": len(result.induced_edges), "seed_nodes": len(result.seed_nodes), "radius": result.radius, "budget": result.budget, "total_candidates": result.total_candidates, "execution_time_ms": result.execution_time_ms, }, } except Exception as e: logger.error(f"Error getting context for symbols: {e}") raise HTTPException(500, f"Failed to get context: {str(e)}") @app.get("/graph/search") async def graph_search( request: Request, q: str, expansion_radius: int = 1, max_context_nodes: int = 50, semantic: bool = False, limit: int = 20, current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Search with graph-based context expansion.""" if dispatcher is None: raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) try: results = list( dispatcher.graph_search( ctx, query=q, expansion_radius=expansion_radius, max_context_nodes=max_context_nodes, semantic=semantic, limit=limit, ) ) # Separate context results from search results search_results = [r for r in results if not r.get("context", False)] context_results = [r for r in results if r.get("context", False)] return { "query": q, "search_results": search_results, "context_results": context_results, "statistics": { "total_results": len(results), "search_results": len(search_results), "context_results": len(context_results), "expansion_radius": expansion_radius, "max_context_nodes": max_context_nodes, }, } except Exception as e: logger.error(f"Error in graph search: {e}") raise HTTPException(500, f"Graph search failed: {str(e)}") @app.get("/graph/status") async def get_graph_status( request: Request, current_user: User = Depends(require_permission(Permission.READ)), ) -> Dict[str, Any]: """Get graph analysis system status.""" if dispatcher is None: raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) try: from .graph import CHUNKER_AVAILABLE status: Dict[str, Any] = {"available": CHUNKER_AVAILABLE} if hasattr(dispatcher, "get_runtime_feature_status"): feature_status = dispatcher.get_runtime_feature_status(ctx).get("graph", {}) status.update(feature_status) if not CHUNKER_AVAILABLE: status["initialized"] = False status["message"] = ( "Install treesitter-chunker for graph features: pip install treesitter-chunker" ) else: # Probe availability via a zero-cost health check; graph init is lazy. health = dispatcher.health_check(ctx) status["initialized"] = health.get("graph_initialized", False) if not status["initialized"]: status.setdefault("status", "unavailable") status.setdefault("reason", "graph_not_initialized") return status except Exception as e: logger.error(f"Error getting graph status: {e}") raise HTTPException(500, f"Failed to get graph status: {str(e)}") @app.post("/graph/initialize") async def initialize_graph( request: Request, file_paths: Optional[List[str]] = None, current_user: User = Depends(require_role(UserRole.ADMIN)), ) -> Dict[str, Any]: """Initialize or rebuild the graph from files (admin only).""" if dispatcher is None: raise HTTPException(503, "Dispatcher not ready") ctx = get_repo_ctx(request) try: # If no file paths provided, scan all indexed files if file_paths is None: if ctx.sqlite_store: files = ctx.sqlite_store.get_all_files() file_paths = [f["path"] for f in files] else: raise HTTPException(400, "No file paths provided and no store available") # Trigger graph initialization via a bulk index_directory call as a proxy. # The dispatcher's graph is built lazily; priming it via index_directory # ensures graph structures are populated for the listed paths. if hasattr(dispatcher, "_ensure_graph_initialized"): success = dispatcher._ensure_graph_initialized(file_paths, ctx=ctx) if not success: return { "status": "failed", "message": "Graph initialization failed. Check that treesitter-chunker is installed.", } else: return { "status": "unavailable", "message": "Graph initialization not supported by the current dispatcher.", } health = dispatcher.health_check(ctx) return { "status": "success", "message": f"Graph initialized from {len(file_paths)} files", "graph_initialized": health.get("graph_initialized", False), } except Exception as e: logger.error(f"Error initializing graph: {e}") raise HTTPException(500, f"Failed to initialize graph: {str(e)}")
import logging
import os
import sys
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
from fastapi import Depends, FastAPI, HTTPException, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from .artifacts.semantic_profiles import SemanticProfileRegistry
from .cache import (
CacheManagerFactory,
QueryCacheConfig,
QueryResultCache,
QueryType,
)
from .cli.index_management import _get_profile_collection_name
from .cli.preflight_commands import format_preflight_report, run_startup_preflight
from .client import (
ClientValidationError,
build_search_options,
execute_search_service,
search_result_for_gateway,
)
from .config.environment import is_production as _is_production
from .config.settings import get_settings
from .config.validation import (
WEAK_CREDENTIAL_BLOCKLIST,
render_validation_errors_to_stderr,
validate_production_config,
)
from .core import RepoContext, RepoResolver
from .core.logging import setup_logging
from .dispatcher.dispatcher_enhanced import EnhancedDispatcher
from .health.repository_readiness import RepositoryReadiness, RepositoryReadinessState
from .indexer.bm25_indexer import BM25Indexer
from .indexer.hybrid_search import HybridSearch, HybridSearchConfig
from .indexing.source_metadata import normalize_friction_category
from .metrics import get_health_checker, get_metrics_collector
from .metrics.middleware import get_business_metrics, setup_metrics_middleware
from .metrics.prometheus_exporter import get_prometheus_exporter
from .plugin_base import SymbolDef
from .plugin_system import PluginManager
from .security import (
AuthCredentials,
AuthenticationMiddleware,
AuthManager,
AuthorizationMiddleware,
Permission,
RateLimitMiddleware,
RequestValidationMiddleware,
SecurityConfig,
SecurityHeadersMiddleware,
User,
UserRole,
get_current_active_user,
require_auth,
require_permission,
require_role,
)
from .security.path_guard import PathTraversalError, PathTraversalGuard
from .security.security_middleware import SecretRedactionResponseMiddleware
from .security.token_validator import TokenValidator
from .setup.qdrant_autostart import ensure_qdrant_running
from .setup.semantic_preflight import run_semantic_preflight
from .storage.sqlite_store import SQLiteStore
from .storage.store_registry import StoreRegistry
from .utils.fuzzy_indexer import FuzzyIndexer
from .utils.index_discovery import IndexDiscovery
from .utils.language_detector import detect_repository_languages
from .watcher.ref_poller import RefPoller
from .watcher_multi_repo import MultiRepositoryWatcher
# Set up logging
setup_logging(log_level="INFO")
logger = logging.getLogger(__name__)
def _parse_cors_origins_from_env() -> list[str]:
"""Parse CORS_ORIGINS env var into a cleaned list of origins.
No default — an unset or empty value means "no cross-origin requests
allowed" and emits a warning. Accepts comma-separated origins. Explicitly
refuses "*" which would reintroduce the old footgun.
"""
raw = os.getenv("CORS_ORIGINS", "").strip()
if not raw:
logger.warning(
"CORS_ORIGINS is unset — no cross-origin requests will be allowed. "
"Set CORS_ORIGINS to a comma-separated allowlist (never '*' in production)."
)
return []
origins = [o.strip() for o in raw.split(",") if o.strip()]
if "*" in origins:
logger.warning(
"CORS_ORIGINS contains '*' — refusing wildcard origin. "
"Replace with an explicit allowlist."
)

Xet Storage Details

Size:
120 kB
·
Xet hash:
38d364b7a4c39a8aa8b9854592d66098cfecf731967b0dcabf47fe80934ea0b6

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.