Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /symbol_graph.py
| """Shared symbol graph extraction built on enriched chunk metadata.""" | |
| from __future__ import annotations | |
| import re | |
| from collections import defaultdict | |
| from fnmatch import fnmatchcase | |
| from pathlib import Path | |
| from typing import Any | |
| from .auto import ZeroConfigAPI | |
| from .core import chunk_file | |
| from .types import RESOLUTION_MODES, ResolutionMode, ResolutionStatus | |
| SKIP_PARTS = { | |
| "__pycache__", | |
| ".git", | |
| ".venv", | |
| "venv", | |
| "node_modules", | |
| ".tox", | |
| ".pytest_cache", | |
| "build", | |
| "dist", | |
| ".eggs", | |
| "archive", | |
| } | |
| CLASS_KINDS = {"class", "interface", "struct", "enum", "trait", "type"} | |
| FUNCTION_KINDS = {"function", "method", "constructor"} | |
| SYMBOL_KINDS = CLASS_KINDS | FUNCTION_KINDS | {"module"} | |
| _IMPORT_KEYWORDS = { | |
| "import", | |
| "from", | |
| "as", | |
| "require", | |
| "package", | |
| "using", | |
| "namespace", | |
| "static", | |
| "new", | |
| } | |
| def _candidate_extensions(language: str | None) -> set[str]: | |
| extension_map = ZeroConfigAPI.EXTENSION_MAP | |
| if language is None: | |
| return set(extension_map) | |
| normalized = language.lower() | |
| return {ext for ext, lang in extension_map.items() if lang == normalized} | |
| def _should_skip_path(path: Path) -> bool: | |
| return any(part in SKIP_PARTS for part in path.parts) | |
| def collect_source_files(path: str | Path, language: str | None = None) -> list[Path]: | |
| """Collect source files for a given path/language selection.""" | |
| root = Path(path) | |
| if not root.exists(): | |
| return [] | |
| extensions = _candidate_extensions(language) | |
| if root.is_file(): | |
| if language is None or root.suffix.lower() in extensions: | |
| return [root] | |
| return [] | |
| files = [ | |
| file_path | |
| for file_path in root.rglob("*") | |
| if file_path.is_file() | |
| and not _should_skip_path(file_path) | |
| and file_path.suffix.lower() in extensions | |
| ] | |
| return sorted(files) | |
| def _detect_language( | |
| file_path: Path, fallback_language: str | None = None | |
| ) -> str | None: | |
| if fallback_language: | |
| return fallback_language.lower() | |
| return ZeroConfigAPI.EXTENSION_MAP.get(file_path.suffix.lower()) | |
| def _module_name(file_path: Path, root: Path) -> str: | |
| try: | |
| rel_path = file_path.relative_to(root if root.is_dir() else root.parent) | |
| except ValueError: | |
| rel_path = file_path | |
| parts = list(rel_path.parts) | |
| if parts and parts[-1].count("."): | |
| parts[-1] = parts[-1].rsplit(".", maxsplit=1)[0] | |
| if parts and parts[-1] == "__init__": | |
| parts = parts[:-1] | |
| return ".".join(parts) if parts else file_path.stem | |
| def _display_file(file_path: Path, root: Path) -> str: | |
| try: | |
| return str(file_path.relative_to(root if root.is_dir() else root.parent)) | |
| except ValueError: | |
| return str(file_path) | |
| def _symbol_id( | |
| module_name: str, qualified_name: str | None, symbol: str | None, fallback: str | |
| ) -> str: | |
| if module_name and qualified_name: | |
| return f"{module_name}:{qualified_name}" | |
| if module_name and symbol: | |
| return f"{module_name}:{symbol}" | |
| return fallback | |
| def _normalize_reference(value: object) -> str | None: | |
| if isinstance(value, dict): | |
| for key in ("qualified_name", "symbol", "name", "module", "path"): | |
| candidate = value.get(key) | |
| if isinstance(candidate, str) and candidate.strip(): | |
| return candidate.strip() | |
| return None | |
| if isinstance(value, str) and value.strip(): | |
| return value.strip() | |
| return None | |
| def _reference_candidates(reference: str) -> list[str]: | |
| tokens = [ | |
| token for token in re.findall(r"[A-Za-z_][A-Za-z0-9_\.]*", reference) if token | |
| ] | |
| candidates: list[str] = [] | |
| for token in tokens: | |
| lowered = token.lower() | |
| if lowered in _IMPORT_KEYWORDS: | |
| continue | |
| candidates.append(token) | |
| if "." in token: | |
| candidates.extend(part for part in token.split(".") if part) | |
| candidates.append(reference) | |
| seen = set() | |
| ordered = [] | |
| for candidate in candidates: | |
| if candidate not in seen: | |
| seen.add(candidate) | |
| ordered.append(candidate) | |
| return ordered | |
| def _build_resolution_indexes( | |
| symbol_lookup: dict[str, dict[str, Any]], | |
| ) -> tuple[dict[str, set[str]], dict[str, set[str]]]: | |
| by_qualified: defaultdict[str, set[str]] = defaultdict(set) | |
| by_name: defaultdict[str, set[str]] = defaultdict(set) | |
| for symbol_id, symbol in symbol_lookup.items(): | |
| qualified_name = symbol.get("qualified_name") | |
| if qualified_name: | |
| by_qualified[str(qualified_name)].add(symbol_id) | |
| name = symbol.get("name") | |
| if name: | |
| by_name[str(name)].add(symbol_id) | |
| return dict(by_qualified), dict(by_name) | |
| def _resolve_reference_candidates( | |
| reference: str, | |
| by_qualified: dict[str, set[str]], | |
| by_name: dict[str, set[str]], | |
| ) -> list[str]: | |
| for candidate in _reference_candidates(reference): | |
| if candidate in by_qualified: | |
| return sorted(by_qualified[candidate]) | |
| candidates: set[str] = set() | |
| for candidate in _reference_candidates(reference): | |
| candidates.update(by_name.get(candidate, set())) | |
| return sorted(candidates) | |
| def _resolution_status(candidates: list[str]) -> ResolutionStatus: | |
| if len(candidates) == 1: | |
| return "resolved" | |
| if candidates: | |
| return "ambiguous" | |
| return "unresolved" | |
| def _import_record(import_text: str, line: int) -> dict[str, Any]: | |
| names = [ | |
| candidate | |
| for candidate in _reference_candidates(import_text) | |
| if "." not in candidate | |
| ] | |
| return { | |
| "module": import_text, | |
| "names": names, | |
| "alias": "", | |
| "line": line, | |
| "is_from_import": import_text.lstrip().startswith("from "), | |
| } | |
| def extract_symbol_facts_for_file( | |
| file_path: Path, | |
| root: Path, | |
| language: str | None = None, | |
| *, | |
| fail_fast: bool = False, | |
| ) -> dict[str, Any]: | |
| """Extract deterministic per-file symbol facts used by graph assembly.""" | |
| detected_language = _detect_language(file_path, language) | |
| display_file = _display_file(file_path, root) | |
| module_name = _module_name(file_path, root) | |
| facts: dict[str, Any] = { | |
| "path": display_file, | |
| "display_file": display_file, | |
| "module": module_name, | |
| "language": detected_language, | |
| "symbol_lookup": {}, | |
| "import_strings": [], | |
| "imports": [], | |
| "chunk_records": [], | |
| "errors": [], | |
| } | |
| if not detected_language: | |
| return facts | |
| try: | |
| chunks = chunk_file( | |
| file_path, | |
| detected_language, | |
| extract_metadata=True, | |
| include_retrieval_metadata=True, | |
| ) | |
| except Exception as exc: # pragma: no cover - defensive around parser failures | |
| if fail_fast: | |
| raise | |
| facts["errors"].append(f"Error extracting symbols from {file_path}: {exc}") | |
| return facts | |
| symbol_lookup: dict[str, dict[str, Any]] = {} | |
| fact_import_strings: list[str] = [] | |
| imports_output: list[dict[str, Any]] = [] | |
| chunk_records: list[dict[str, Any]] = [] | |
| for chunk in chunks: | |
| metadata = chunk.metadata or {} | |
| fact_import_strings.extend( | |
| str(value) for value in metadata.get("imports", []) if str(value) | |
| ) | |
| symbol = metadata.get("symbol") | |
| kind = metadata.get("kind") | |
| if not symbol or not kind or kind not in SYMBOL_KINDS: | |
| continue | |
| qualified_name = metadata.get("qualified_name") | |
| symbol_id = _symbol_id( | |
| module_name, | |
| str(qualified_name) if qualified_name else None, | |
| str(symbol), | |
| chunk.definition_id or chunk.node_id, | |
| ) | |
| symbol_record = { | |
| "name": str(symbol), | |
| "kind": str(kind), | |
| "file": display_file, | |
| "line": chunk.start_line, | |
| "end_line": chunk.end_line, | |
| "module": module_name, | |
| "parent_class": str(metadata.get("parent_symbol") or ""), | |
| "qualified_name": qualified_name, | |
| "signature": metadata.get("signature_text"), | |
| "semantic_path": metadata.get("semantic_path"), | |
| "language": detected_language, | |
| } | |
| symbol_lookup[symbol_id] = symbol_record | |
| import_strings = [ | |
| str(value) for value in metadata.get("imports", []) if str(value) | |
| ] | |
| imports_output.extend( | |
| _import_record(import_text, chunk.start_line) | |
| for import_text in import_strings | |
| ) | |
| chunk_records.append( | |
| { | |
| "context": { | |
| "id": symbol_id, | |
| "line": chunk.start_line, | |
| "file": display_file, | |
| }, | |
| "imports": import_strings, | |
| "dependencies": [ | |
| str(value) | |
| for value in metadata.get("dependencies", []) | |
| if str(value) | |
| ], | |
| "calls": ( | |
| list(metadata.get("calls", [])) | |
| if isinstance(metadata.get("calls"), list) | |
| else [] | |
| ), | |
| } | |
| ) | |
| facts["symbol_lookup"] = symbol_lookup | |
| facts["import_strings"] = sorted(dict.fromkeys(fact_import_strings)) | |
| facts["imports"] = imports_output | |
| facts["chunk_records"] = chunk_records | |
| return facts | |
| def assemble_symbol_graph( | |
| facts: list[dict[str, Any]], | |
| *, | |
| total_files: int | None = None, | |
| no_source_error: str | None = None, | |
| resolution_mode: ResolutionMode = "permissive", | |
| ) -> dict[str, Any]: | |
| """Assemble public symbol graph output from per-file facts.""" | |
| if resolution_mode not in RESOLUTION_MODES: | |
| msg = f"Unsupported resolution_mode: {resolution_mode}" | |
| raise ValueError(msg) | |
| if total_files == 0 and no_source_error: | |
| return { | |
| "symbols": {"classes": [], "functions": [], "imports": []}, | |
| "relationships": [], | |
| "metadata": { | |
| "files_processed": 0, | |
| "total_classes": 0, | |
| "total_functions": 0, | |
| "total_imports": 0, | |
| "total_relationships": 0, | |
| "errors": [no_source_error], | |
| }, | |
| "symbol_lookup": {}, | |
| "errors": [no_source_error], | |
| } | |
| symbol_lookup: dict[str, dict[str, Any]] = {} | |
| imports_output: list[dict[str, Any]] = [] | |
| chunk_records: list[dict[str, Any]] = [] | |
| errors: list[str] = [] | |
| for file_facts in facts: | |
| symbol_lookup.update(file_facts.get("symbol_lookup", {})) | |
| imports_output.extend(file_facts.get("imports", [])) | |
| chunk_records.extend(file_facts.get("chunk_records", [])) | |
| errors.extend(str(error) for error in file_facts.get("errors", [])) | |
| by_qualified, by_name = _build_resolution_indexes(symbol_lookup) | |
| relationships: list[dict[str, Any]] = [] | |
| seen_relationships: set[tuple[str, str, str]] = set() | |
| for chunk_record in chunk_records: | |
| relationship_context = chunk_record["context"] | |
| from_id = relationship_context["id"] | |
| line = relationship_context["line"] | |
| file_name = relationship_context["file"] | |
| for relationship_type, values in ( | |
| ("imports", chunk_record.get("imports", [])), | |
| ("dependencies", chunk_record.get("dependencies", [])), | |
| ("calls", chunk_record.get("calls", [])), | |
| ): | |
| for value in values: | |
| reference = _normalize_reference(value) | |
| if not reference: | |
| continue | |
| candidates = _resolve_reference_candidates( | |
| reference, by_qualified, by_name | |
| ) | |
| resolution = _resolution_status(candidates) | |
| to_id = candidates[0] if resolution == "resolved" else reference | |
| if to_id == from_id: | |
| continue | |
| dedupe_key = (from_id, to_id, relationship_type) | |
| if dedupe_key in seen_relationships: | |
| continue | |
| seen_relationships.add(dedupe_key) | |
| relationships.append( | |
| { | |
| "from": from_id, | |
| "to": to_id, | |
| "type": relationship_type, | |
| "line": line, | |
| "file": file_name, | |
| "is_internal": resolution == "resolved", | |
| "reference": reference, | |
| "resolution": resolution, | |
| "candidates": candidates, | |
| "resolution_mode": resolution_mode, | |
| "provenance": { | |
| "resolver": "extract_symbol_graph", | |
| "source": "syntax", | |
| }, | |
| } | |
| ) | |
| class_symbols = [ | |
| symbol for symbol in symbol_lookup.values() if symbol["kind"] in CLASS_KINDS | |
| ] | |
| function_symbols = [ | |
| symbol for symbol in symbol_lookup.values() if symbol["kind"] in FUNCTION_KINDS | |
| ] | |
| total = len(facts) if total_files is None else total_files | |
| metadata = { | |
| "files_processed": total - len(errors), | |
| "total_classes": len(class_symbols), | |
| "total_functions": len(function_symbols), | |
| "total_imports": len(imports_output), | |
| "total_relationships": len(relationships), | |
| "errors": errors, | |
| } | |
| return { | |
| "symbols": { | |
| "classes": class_symbols, | |
| "functions": function_symbols, | |
| "imports": imports_output, | |
| }, | |
| "relationships": relationships, | |
| "metadata": metadata, | |
| "symbol_lookup": symbol_lookup, | |
| "errors": errors, | |
| } | |
| def extract_symbol_graph( | |
| path: str | Path, | |
| language: str | None = None, | |
| resolution_mode: ResolutionMode = "permissive", | |
| fail_fast: bool = False, | |
| ) -> dict[str, Any]: | |
| """Extract a language-agnostic symbol graph from chunk metadata.""" | |
| if resolution_mode not in RESOLUTION_MODES: | |
| msg = f"Unsupported resolution_mode: {resolution_mode}" | |
| raise ValueError(msg) | |
| root = Path(path) | |
| files = collect_source_files(root, language) | |
| if not files: | |
| return { | |
| "symbols": {"classes": [], "functions": [], "imports": []}, | |
| "relationships": [], | |
| "metadata": { | |
| "files_processed": 0, | |
| "total_classes": 0, | |
| "total_functions": 0, | |
| "total_imports": 0, | |
| "total_relationships": 0, | |
| "errors": [f"No source files found in {root}"], | |
| }, | |
| "symbol_lookup": {}, | |
| "errors": [f"No source files found in {root}"], | |
| } | |
| facts = [ | |
| extract_symbol_facts_for_file( | |
| file_path, | |
| root, | |
| language, | |
| fail_fast=fail_fast, | |
| ) | |
| for file_path in files | |
| ] | |
| return assemble_symbol_graph( | |
| facts, | |
| total_files=len(files), | |
| resolution_mode=resolution_mode, | |
| ) | |
| def find_symbols( | |
| path: str | Path, pattern: str, kind: str | None = None | |
| ) -> list[dict[str, Any]]: | |
| """Find symbols matching a glob-style pattern.""" | |
| graph = extract_symbol_graph(path) | |
| matches = [] | |
| for symbol_id, symbol in graph["symbol_lookup"].items(): | |
| if kind and symbol.get("kind") != kind: | |
| continue | |
| name = str(symbol.get("name") or "") | |
| qualified_name = str(symbol.get("qualified_name") or "") | |
| if fnmatchcase(name, pattern) or ( | |
| qualified_name and fnmatchcase(qualified_name, pattern) | |
| ): | |
| matches.append({"id": symbol_id, **symbol}) | |
| matches.sort( | |
| key=lambda symbol: ( | |
| symbol.get("file", ""), | |
| symbol.get("line", 0), | |
| symbol.get("name", ""), | |
| ) | |
| ) | |
| return matches | |
Xet Storage Details
- Size:
- 16 kB
- Xet hash:
- 36c8fb2317cf27148c7d3d614f93003e7166975c006ff07b04e6e47572970759
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.