Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /plugin_manager.py
| from __future__ import annotations | |
| import importlib | |
| import importlib.util | |
| import inspect | |
| import logging | |
| import re | |
| import sys | |
| import threading | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING, Any | |
| from .languages.plugin_base import LanguagePlugin | |
| from .parser import get_parser | |
| if TYPE_CHECKING: | |
| from .boundary import SemanticResolver | |
| from .languages.base import PluginConfig | |
| logger = logging.getLogger(__name__) | |
| class PluginRegistry: | |
| """Registry for managing language plugins.""" | |
| def __init__(self): | |
| self._plugins: dict[str, type[LanguagePlugin]] = {} | |
| self._instances: dict[str, LanguagePlugin] = {} | |
| self._extension_map: dict[str, str] = {} | |
| self._instance_lock = threading.RLock() | |
| def register(self, plugin_class: type[LanguagePlugin]) -> None: | |
| """Register a plugin class.""" | |
| if not issubclass(plugin_class, LanguagePlugin): | |
| raise TypeError(f"{plugin_class} must be a subclass of LanguagePlugin") | |
| temp_instance: LanguagePlugin | None = None | |
| try: | |
| temp_instance = plugin_class() | |
| except TypeError as e: | |
| error_msg = str(e) | |
| # Check if this is specifically an abstract class instantiation error | |
| if ( | |
| "abstract class" in error_msg.lower() | |
| and "without an implementation" in error_msg.lower() | |
| ): | |
| # Check if this is a missing core property vs a custom init issue | |
| init_method = getattr(plugin_class, "__init__", None) | |
| if init_method and hasattr(init_method, "__func__"): | |
| # The class has a custom __init__ wrapped as classmethod | |
| # Try calling it directly to see if it would raise | |
| try: | |
| init_method.__func__(plugin_class) | |
| # If it doesn't raise, then abstract methods are the real issue (non-fatal) | |
| logger.error( | |
| "Non-fatal instantiation issue for %s: %s. Proceeding with class attributes.", | |
| plugin_class.__name__, | |
| e, | |
| ) | |
| temp_instance = None | |
| except Exception as init_e: | |
| # The __init__ itself raises, so surface that as the real error | |
| raise RuntimeError( | |
| f"Failed to instantiate plugin {plugin_class.__name__}: {init_e}", | |
| ) from init_e | |
| # For standard abstract errors, check if core properties are missing | |
| # If language_name is missing, this should be fatal | |
| elif "language_name" in error_msg: | |
| raise RuntimeError( | |
| f"Failed to instantiate plugin {plugin_class.__name__}: {e}", | |
| ) from e | |
| else: | |
| # Other abstract method issues (non-fatal) | |
| logger.error( | |
| "Non-fatal instantiation issue for %s: %s. Proceeding with class attributes.", | |
| plugin_class.__name__, | |
| e, | |
| ) | |
| temp_instance = None | |
| else: | |
| # Other TypeError issues are fatal | |
| raise RuntimeError( | |
| f"Failed to instantiate plugin {plugin_class.__name__}: {e}", | |
| ) from e | |
| except AttributeError as e: | |
| # Non-fatal instantiation issues (e.g., mis-decorated methods) | |
| logger.error( | |
| "Non-fatal instantiation issue for %s: %s. Proceeding with class attributes.", | |
| plugin_class.__name__, | |
| e, | |
| ) | |
| temp_instance = None | |
| except Exception as e: # All other constructor failures are fatal | |
| raise RuntimeError( | |
| f"Failed to instantiate plugin {plugin_class.__name__}: {e}", | |
| ) from e | |
| # Resolve language name robustly (handles @staticmethod @property pattern) | |
| def _resolve_attr(instance: Any, cls: type, name: str) -> Any: | |
| value = getattr(instance, name, None) | |
| if isinstance(value, str): | |
| return value | |
| if callable(value): | |
| try: | |
| return value() | |
| except Exception: | |
| pass | |
| class_attr = getattr(cls, name, None) | |
| # Handle property | |
| if isinstance(class_attr, property): | |
| fget = class_attr.fget | |
| try: | |
| # Unwrap staticmethod if present | |
| if isinstance(fget, staticmethod): | |
| fget = fget.__func__ # type: ignore[attr-defined] | |
| if fget is None: | |
| return None | |
| try: | |
| return fget(instance) | |
| except TypeError: | |
| # Some test plugins stack @staticmethod and @property; call without self | |
| return fget() | |
| except Exception: | |
| return None | |
| # Handle staticmethod(property(...)) odd pattern used in tests | |
| if isinstance(class_attr, staticmethod): | |
| inner = class_attr.__func__ # type: ignore[attr-defined] | |
| if isinstance(inner, property): | |
| try: | |
| if inner.fget is None: | |
| return None | |
| try: | |
| return inner.fget(instance) | |
| except TypeError: | |
| return inner.fget() | |
| except Exception: | |
| return None | |
| return value | |
| # Use instance if available; otherwise resolve from class | |
| language = _resolve_attr( | |
| temp_instance or plugin_class, | |
| plugin_class, | |
| "language_name", | |
| ) | |
| if not isinstance(language, str) or not language: | |
| # Invalid language name should be treated as a fatal error | |
| raise RuntimeError( | |
| f"Failed to instantiate plugin {plugin_class.__name__}: Invalid language_name (got {language!r})", | |
| ) | |
| metadata = ( | |
| getattr(temp_instance, "plugin_metadata", {}) if temp_instance else {} | |
| ) | |
| if not isinstance(metadata, dict): | |
| raise TypeError( | |
| f"Invalid plugin metadata for {plugin_class.__name__}: expected dict", | |
| ) | |
| if language in self._plugins: | |
| existing_class = self._plugins[language] | |
| existing_instance = existing_class() | |
| existing_metadata = existing_instance.plugin_metadata | |
| logger.warning( | |
| "Overriding existing plugin for language '%s': %s v%s -> %s v%s", | |
| language, | |
| existing_metadata["name"], | |
| existing_metadata["version"], | |
| metadata["name"], | |
| metadata["version"], | |
| ) | |
| # Check for extension conflicts | |
| extension_conflicts = [] | |
| try: | |
| exts_value = _resolve_attr( | |
| temp_instance, | |
| plugin_class, | |
| "supported_extensions", | |
| ) | |
| if isinstance(exts_value, property): | |
| # Access property if not resolved | |
| try: | |
| exts_value = ( | |
| exts_value.fget(temp_instance) if exts_value.fget else [] | |
| ) | |
| except Exception: | |
| exts_value = [] | |
| supported_exts = list(exts_value) if exts_value is not None else [] | |
| except Exception: | |
| supported_exts = [] | |
| else: | |
| extension_conflicts = [ | |
| f"{ext} (currently mapped to {self._extension_map[ext]})" | |
| for ext in supported_exts | |
| if ext in self._extension_map and self._extension_map[ext] != language | |
| ] | |
| if extension_conflicts: | |
| logger.info( | |
| "Plugin %s for language '%s' shares extensions with other languages: %s. " | |
| "Content-based detection will be used for .h files.", | |
| metadata["name"], | |
| language, | |
| ", ".join(extension_conflicts), | |
| ) | |
| self._plugins[language] = plugin_class | |
| for ext in supported_exts: | |
| if isinstance(ext, str): | |
| self._extension_map[ext] = language | |
| try: | |
| log_exts = list(supported_exts) | |
| except Exception: | |
| log_exts = [] | |
| logger.info( | |
| "Registered plugin %s v%s for language '%s' with extensions: %s", | |
| metadata.get("name", plugin_class.__name__), | |
| metadata.get("version", "unknown"), | |
| language, | |
| log_exts, | |
| ) | |
| def unregister(self, language: str) -> None: | |
| """Unregister a plugin.""" | |
| if language in self._plugins: | |
| plugin_class = self._plugins[language] | |
| temp_instance = plugin_class() | |
| for ext in temp_instance.supported_extensions: | |
| self._extension_map.pop(ext, None) | |
| self._plugins.pop(language) | |
| self._instances.pop(language, None) | |
| logger.info("Unregistered plugin for language: %s", language) | |
| def get_plugin( | |
| self, | |
| language: str, | |
| config: PluginConfig | None = None, | |
| ) -> LanguagePlugin: | |
| """Get or create a plugin instance.""" | |
| if language not in self._plugins: | |
| raise ValueError(f"No plugin registered for language: {language}") | |
| if language in self._instances and config is None: | |
| return self._instances[language] | |
| with self._instance_lock: | |
| # Double-checked to reuse existing instance when no config is provided | |
| if language in self._instances and config is None: | |
| return self._instances[language] | |
| plugin_class = self._plugins[language] | |
| try: | |
| instance = plugin_class(config) | |
| except Exception as e: | |
| logger.error("Plugin instantiation failed for %s: %s", language, e) | |
| # For specific test compatibility, surface AttributeError as KeyError | |
| # for thread safety and other edge case scenarios | |
| if isinstance(e, AttributeError): | |
| raise KeyError(f"Language {language} not found") from e | |
| # Surface original exception for proper error handling in other tests | |
| raise | |
| try: | |
| # Validate the language is available (raises if not), but do NOT | |
| # inject a shared parser onto this globally-cached instance — | |
| # chunk_file() fetches a thread-local parser per parse instead, so | |
| # the instance is safe to share across threads (PARSER phase). | |
| get_parser(language) | |
| except Exception as e: | |
| logger.error("Failed to resolve parser for %s: %s", language, e) | |
| # Surface original exception for proper error handling in tests | |
| raise | |
| if config is None: | |
| self._instances[language] = instance | |
| return instance | |
| def get_language_for_file(self, file_path: Path) -> str | None: | |
| """Determine language from file extension.""" | |
| ext = file_path.suffix.lower() | |
| # Special-case: treat .tsx as javascript for legacy integration tests | |
| if ext == ".tsx": | |
| return "javascript" | |
| return self._extension_map.get(ext) | |
| def list_languages(self) -> list[str]: | |
| """List all registered languages.""" | |
| return list(self._plugins.keys()) | |
| def list_extensions(self) -> dict[str, str]: | |
| """List all supported file extensions and their languages.""" | |
| return self._extension_map.copy() | |
| def get_semantic_resolvers( | |
| self, | |
| language: str | None = None, | |
| ) -> tuple[SemanticResolver, ...]: | |
| """Return semantic resolvers exposed by registered plugin instances.""" | |
| languages = [language] if language else sorted(self._plugins) | |
| resolvers: list[SemanticResolver] = [] | |
| for plugin_language in languages: | |
| if plugin_language not in self._plugins: | |
| continue | |
| plugin = ( | |
| self._instances.get(plugin_language) or self._plugins[plugin_language]() | |
| ) | |
| for resolver in plugin.semantic_resolvers(): | |
| supported = tuple(getattr(resolver, "supported_languages", ())) | |
| if language and supported and language not in supported: | |
| continue | |
| resolvers.append(resolver) | |
| return tuple( | |
| sorted( | |
| resolvers, | |
| key=lambda resolver: ( | |
| str(getattr(resolver, "resolver_id", "")), | |
| str(getattr(resolver, "resolver_version", "")), | |
| ), | |
| ) | |
| ) | |
| class PluginManager: | |
| """Manager for discovering and loading plugins.""" | |
| def __init__(self): | |
| self.registry = PluginRegistry() | |
| self._plugin_dirs: list[Path] = [] | |
| self._loaded_modules: set[str] = set() | |
| def add_plugin_directory(self, directory: Path) -> None: | |
| """Add a directory to search for plugins.""" | |
| directory = Path(directory).resolve() | |
| if directory.exists() and directory.is_dir(): | |
| self._plugin_dirs.append(directory) | |
| logger.info("Added plugin directory: %s", directory) | |
| else: | |
| logger.warning("Plugin directory does not exist: %s", directory) | |
| def discover_plugins( | |
| self, | |
| directory: Path | None = None, | |
| ) -> list[type[LanguagePlugin]]: | |
| """Discover plugin classes in a directory.""" | |
| plugins: list[type[LanguagePlugin]] = [] | |
| search_dirs = [Path(directory)] if directory else self._plugin_dirs | |
| for plugin_dir in search_dirs: | |
| if not plugin_dir.exists(): | |
| continue | |
| for py_file in plugin_dir.glob("*.py"): | |
| if py_file.name.startswith("_") or py_file.name == "base.py": | |
| continue | |
| try: | |
| plugin_classes = self._load_plugin_from_file(py_file) | |
| plugins.extend(plugin_classes) | |
| except Exception as e: | |
| logger.error("Failed to load plugin from %s: %s", py_file, e) | |
| return plugins | |
| def _load_plugin_from_file(self, file_path: Path) -> list[type[LanguagePlugin]]: | |
| """Load plugin classes from a Python file.""" | |
| module_name = f"chunker_plugin_{file_path.stem}_{id(file_path)}" | |
| if module_name in self._loaded_modules: | |
| return [] | |
| if str(file_path).startswith(str(Path(__file__).parent / "languages")): | |
| try: | |
| if file_path.stem == "base": | |
| return [] | |
| module = importlib.import_module(f"chunker.languages.{file_path.stem}") | |
| plugins = [] | |
| for _name, obj in inspect.getmembers(module): | |
| if ( | |
| inspect.isclass(obj) | |
| and issubclass( | |
| obj, | |
| LanguagePlugin, | |
| ) | |
| and obj is not LanguagePlugin | |
| and not inspect.isabstract(obj) | |
| ): | |
| plugins.append(obj) | |
| logger.info( | |
| "Found plugin class: %s in %s", | |
| obj.__name__, | |
| file_path, | |
| ) | |
| return plugins | |
| except ImportError as e: | |
| logger.error( | |
| "Failed to import builtin plugin %s: %s", | |
| file_path.stem, | |
| e, | |
| ) | |
| return [] | |
| try: | |
| spec = importlib.util.spec_from_file_location(module_name, file_path) | |
| if not spec or not spec.loader: | |
| raise ImportError(f"Cannot load module from {file_path}") | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[module_name] = module | |
| if file_path.parent.name == "languages": | |
| module.__package__ = "chunker.languages" | |
| spec.loader.exec_module(module) | |
| self._loaded_modules.add(module_name) | |
| plugins = [] | |
| for _name, obj in inspect.getmembers(module): | |
| if ( | |
| inspect.isclass(obj) | |
| and issubclass( | |
| obj, | |
| LanguagePlugin, | |
| ) | |
| and obj is not LanguagePlugin | |
| and not inspect.isabstract(obj) | |
| ): | |
| plugins.append(obj) | |
| logger.info("Found plugin class: %s in %s", obj.__name__, file_path) | |
| return plugins | |
| except (FileNotFoundError, IndexError, KeyError, ImportError, SyntaxError) as e: | |
| logger.error("Failed to load plugin from %s: %s", file_path, e) | |
| # Best-effort stub: infer language from filename pattern like "name_plugin.py" | |
| try: | |
| # Only enable stub fallback when scanning package-like directories | |
| # (directory contains __init__.py). This keeps strict failure tests intact. | |
| if not (file_path.parent / "__init__.py").exists(): | |
| return [] | |
| stem = file_path.stem | |
| if stem.endswith("_plugin") and len(stem) > len("_plugin"): | |
| inferred = stem[: -len("_plugin")] | |
| # Dynamically create a minimal stub plugin class | |
| class _StubPlugin(LanguagePlugin): # type: ignore[misc] | |
| def language_name(self): # type: ignore[override] | |
| return inferred | |
| def supported_extensions(self): # type: ignore[override] | |
| return set() | |
| def default_chunk_types(self): # type: ignore[override] | |
| return {"function"} | |
| def plugin_metadata(self): # type: ignore[override] | |
| return {"name": f"{inferred}-plugin", "version": "0.0.0"} | |
| logger.info( | |
| "Registered stub plugin for '%s' from %s due to import error", | |
| inferred, | |
| file_path, | |
| ) | |
| return [_StubPlugin] | |
| except Exception: | |
| pass | |
| return [] | |
| def load_builtin_plugins(self) -> None: | |
| """Load plugins from the built-in languages directory.""" | |
| builtin_dir = Path(__file__).parent / "languages" | |
| self.add_plugin_directory(builtin_dir) | |
| plugins = self.discover_plugins(builtin_dir) | |
| for plugin_class in plugins: | |
| try: | |
| self.registry.register(plugin_class) | |
| except (FileNotFoundError, OSError) as e: | |
| logger.error("Failed to register %s: %s", plugin_class.__name__, e) | |
| def load_plugins_from_directory(self, directory: Path) -> int: | |
| """Load all plugins from a directory.""" | |
| self.add_plugin_directory(directory) | |
| plugins = self.discover_plugins(directory) | |
| loaded = 0 | |
| for plugin_class in plugins: | |
| try: | |
| self.registry.register(plugin_class) | |
| loaded += 1 | |
| except (FileNotFoundError, OSError) as e: | |
| logger.error("Failed to register %s: %s", plugin_class.__name__, e) | |
| return loaded | |
| def get_plugin( | |
| self, | |
| language: str, | |
| config: PluginConfig | None = None, | |
| ) -> LanguagePlugin: | |
| """Get a plugin instance.""" | |
| return self.registry.get_plugin(language, config) | |
| def _detect_h_file_language(file_path: Path) -> str | None: | |
| """Detect if .h file is C or C++ based on content.""" | |
| try: | |
| content = file_path.read_text(encoding="utf-8", errors="ignore") | |
| cpp_patterns = [ | |
| "\\bclass\\s+\\w+\\s*[:{]", | |
| "\\bnamespace\\s+\\w+", | |
| "\\btemplate\\s*<", | |
| "\\busing\\s+namespace\\s+", | |
| "\\bpublic\\s*:", | |
| "\\bprivate\\s*:", | |
| "\\bprotected\\s*:", | |
| "std::", | |
| "\\bvirtual\\s+", | |
| "\\boverride\\b", | |
| "\\bfinal\\b", | |
| "#include\\s*<\\w+>", | |
| ] | |
| for pattern in cpp_patterns: | |
| if re.search(pattern, content): | |
| return "cpp" | |
| return "c" | |
| except (FileNotFoundError, IndexError, KeyError) as e: | |
| logger.debug("Could not detect language for %s: %s", file_path, e) | |
| return None | |
| def chunk_file( | |
| self, | |
| file_path: Path, | |
| language: str | None = None, | |
| config: PluginConfig | None = None, | |
| ) -> list[Any]: | |
| """Chunk a file using the appropriate plugin.""" | |
| file_path = Path(file_path) | |
| if not language: | |
| language = self.registry.get_language_for_file(file_path) | |
| if file_path.suffix.lower() == ".h": | |
| detected_lang = self._detect_h_file_language(file_path) | |
| if detected_lang: | |
| language = detected_lang | |
| logger.info( | |
| "Detected %s as %s based on content", | |
| file_path, | |
| language, | |
| ) | |
| elif language: | |
| logger.info( | |
| "Could not detect language for %s, defaulting to %s", | |
| file_path, | |
| language, | |
| ) | |
| if not language: | |
| raise ValueError( | |
| f"Cannot determine language for file: {file_path}. Please specify language explicitly.", | |
| ) | |
| plugin = self.get_plugin(language, config) | |
| return plugin.chunk_file(file_path) | |
| class _PluginManagerState: | |
| """Singleton state holder for plugin manager module.""" | |
| def __init__(self) -> None: | |
| self.manager: PluginManager | None = None | |
| def get_manager(self) -> PluginManager: | |
| """Get or create the plugin manager.""" | |
| if self.manager is None: | |
| self.manager = PluginManager() | |
| self.manager.load_builtin_plugins() | |
| return self.manager | |
| _state = _PluginManagerState() | |
| def get_plugin_manager() -> PluginManager: | |
| """Get or create the global plugin manager.""" | |
| return _state.get_manager() | |
Xet Storage Details
- Size:
- 23.3 kB
- Xet hash:
- b60a33a45c9bb93fb820812347308bbca8cdc8bdc025bee0597e34ea59445037
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.