Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /smart_context.py
| """Smart context implementation for intelligent chunk context selection.""" | |
| import builtins | |
| import hashlib | |
| import re | |
| import time | |
| from collections import defaultdict | |
| from collections.abc import Iterable | |
| from typing import Any | |
| from .interfaces.smart_context import ( | |
| ContextCache, | |
| ContextMetadata, | |
| ContextStrategy, | |
| SmartContextProvider, | |
| ) | |
| from .parser import get_parser | |
| from .types import CodeChunk | |
| class TreeSitterSmartContextProvider(SmartContextProvider): | |
| """ | |
| Provides intelligent context for chunks using tree-sitter analysis. | |
| This implementation analyzes code structure, dependencies, and relationships | |
| to provide optimal context for LLM processing. | |
| """ | |
| def __init__(self, cache: ContextCache | None = None): | |
| """ | |
| Initialize the smart context provider. | |
| Args: | |
| cache: Optional cache for context computations | |
| """ | |
| self.cache = cache or InMemoryContextCache() | |
| # Back-compat alias used by some tests | |
| self._cache = self.cache | |
| # Memoize per-chunk semantic features by chunk_id so the all-pairs | |
| # similarity scan extracts each chunk's features ONCE, not once per | |
| # (request, candidate) pair -- turns repeated O(n^2) extraction into | |
| # O(n) amortized (COREFIX). | |
| self._feature_cache: dict[str, dict[str, Any]] = {} | |
| def get_semantic_context( | |
| self, | |
| chunk: CodeChunk, | |
| max_tokens: int = 2000, | |
| ) -> tuple[str, ContextMetadata]: | |
| """ | |
| Get semantically relevant context for a chunk. | |
| This includes related functions, classes, or modules that are | |
| semantically connected to the chunk (similar functionality, same domain). | |
| Args: | |
| chunk: The chunk to get context for | |
| max_tokens: Maximum tokens for context | |
| Returns: | |
| Tuple of (context_string, metadata) | |
| """ | |
| chunk_features = self._features_for(chunk) | |
| file_chunks = self._get_file_chunks(chunk.file_path, chunk.language) | |
| # BOUNDED candidate selection (COREFIX): instead of comparing against every | |
| # chunk in the file (O(n^2) across requests), use an inverted identifier | |
| # index to consider only candidates that share >= 1 identifier with the | |
| # query, capped at _MAX_CONTEXT_CANDIDATES. | |
| candidates = self._candidate_subset(chunk, chunk_features, file_chunks) | |
| similar_chunks = [] | |
| for candidate in candidates: | |
| if candidate.chunk_id == chunk.chunk_id: | |
| continue | |
| candidate_features = self._features_for(candidate) | |
| similarity_score = self._calculate_semantic_similarity( | |
| chunk_features, | |
| candidate_features, | |
| ) | |
| if similarity_score > 0.3: | |
| metadata = ContextMetadata( | |
| relevance_score=similarity_score, | |
| relationship_type="semantic", | |
| distance=abs(candidate.start_line - chunk.start_line), | |
| token_count=len(candidate.content.split()), | |
| ) | |
| similar_chunks.append((candidate, metadata)) | |
| similar_chunks.sort(key=lambda x: x[1].relevance_score, reverse=True) | |
| context_parts = [] | |
| total_tokens = 0 | |
| for candidate, _metadata in similar_chunks: | |
| estimated_tokens = len(candidate.content.split()) | |
| if total_tokens + estimated_tokens > max_tokens: | |
| break | |
| context_parts.append( | |
| f"""# Related {candidate.node_type} at line {candidate.start_line}: | |
| {candidate.content}""", | |
| ) | |
| total_tokens += estimated_tokens | |
| context_string = "\n\n".join(context_parts) | |
| overall_metadata = ContextMetadata( | |
| relevance_score=( | |
| similar_chunks[0][1].relevance_score if similar_chunks else 0.0 | |
| ), | |
| relationship_type="semantic", | |
| distance=0, | |
| token_count=total_tokens, | |
| ) | |
| return context_string, overall_metadata | |
| def get_dependency_context( | |
| self, | |
| chunk: CodeChunk, | |
| chunks: list[CodeChunk], | |
| ) -> list[tuple[CodeChunk, ContextMetadata]]: | |
| """ | |
| Get chunks that this chunk depends on. | |
| This includes imports, function calls, class inheritance, etc. | |
| Args: | |
| chunk: The chunk to analyze | |
| chunks: All available chunks to search | |
| Returns: | |
| List of (chunk, metadata) tuples for dependencies | |
| """ | |
| cache_type = self._candidate_cache_type("dependency", chunks) | |
| cached = self.cache.get(chunk.chunk_id, cache_type) | |
| if cached is not None: | |
| return cached | |
| dependencies = [] | |
| imports = self._extract_imports(chunk) | |
| function_calls = self._extract_function_calls(chunk) | |
| class_references = self._extract_class_references(chunk) | |
| for candidate in chunks: | |
| relevance_score = 0.0 | |
| if self._defines_import(candidate, imports): | |
| relevance_score += 0.8 | |
| if self._defines_function(candidate, function_calls): | |
| relevance_score += 0.9 | |
| if self._defines_class(candidate, class_references): | |
| relevance_score += 0.9 | |
| if relevance_score > 0: | |
| metadata = ContextMetadata( | |
| relevance_score=relevance_score, | |
| relationship_type="dependency", | |
| distance=self._calculate_file_distance(chunk, candidate), | |
| token_count=len(candidate.content.split()), | |
| ) | |
| dependencies.append((candidate, metadata)) | |
| dependencies.sort(key=lambda x: x[1].relevance_score, reverse=True) | |
| self.cache.set(chunk.chunk_id, cache_type, dependencies) | |
| return dependencies | |
| def get_usage_context( | |
| self, | |
| chunk: CodeChunk, | |
| chunks: list[CodeChunk], | |
| ) -> list[tuple[CodeChunk, ContextMetadata]]: | |
| """ | |
| Get chunks that use this chunk. | |
| This includes all places where this chunk is imported, called, or referenced. | |
| Args: | |
| chunk: The chunk to analyze | |
| chunks: All available chunks to search | |
| Returns: | |
| List of (chunk, metadata) tuples for usages | |
| """ | |
| cache_type = self._candidate_cache_type("usage", chunks) | |
| cached = self.cache.get(chunk.chunk_id, cache_type) | |
| if cached is not None: | |
| return cached | |
| usages = [] | |
| chunk_exports = self._extract_exports(chunk) | |
| # Heuristic: detect Flask/Django-like route paths for cross-language usage | |
| route_paths: set[str] = set() | |
| if chunk.language == "python" and "@app.route(" in chunk.content: | |
| import re as _re | |
| for m in _re.finditer( | |
| r"@app\.route\(\s*['\"]([^'\"]+)['\"]", | |
| chunk.content, | |
| ): | |
| route_paths.add(m.group(1)) | |
| for candidate in chunks: | |
| if candidate.chunk_id == chunk.chunk_id: | |
| continue | |
| relevance_score = 0.0 | |
| # If candidate mentions the class/function name, consider it usage | |
| name_hint = None | |
| if chunk.node_type in {"function_definition", "class_definition"}: | |
| import re as _re | |
| m = _re.search( | |
| r"\b(def|class)\s+([A-Za-z_][A-Za-z0-9_]*)", | |
| chunk.content, | |
| ) | |
| if m: | |
| name_hint = m.group(2) | |
| if self._imports_from(candidate, chunk, chunk_exports): | |
| relevance_score += 0.8 | |
| if self._calls_functions_from(candidate, chunk_exports): | |
| relevance_score += 0.7 | |
| if self._uses_classes_from(candidate, chunk_exports): | |
| relevance_score += 0.7 | |
| # Cross-language API usage detection: JS fetch/axios referencing the same route | |
| if route_paths and candidate.language in {"javascript", "typescript"}: | |
| text = candidate.content | |
| for route in route_paths: | |
| if ( | |
| route in text | |
| or f"${{this.baseUrl}}{route[len('/api'):] if route.startswith('/api') else route}" | |
| in text | |
| ): | |
| relevance_score += 0.8 | |
| break | |
| # Fallback: If Python route chunk exists and this is JS client code, nudge relevance | |
| if ( | |
| candidate.language in {"javascript", "typescript"} | |
| and chunk.language == "python" | |
| and "@app.route(" in chunk.content | |
| ): | |
| relevance_score += 0.3 | |
| if name_hint and name_hint in candidate.content: | |
| relevance_score += 0.3 | |
| if candidate.file_path == chunk.file_path: | |
| relevance_score += 0.2 | |
| if relevance_score > 0: | |
| metadata = ContextMetadata( | |
| relevance_score=relevance_score, | |
| relationship_type="usage", | |
| distance=self._calculate_file_distance(chunk, candidate), | |
| token_count=len(candidate.content.split()), | |
| ) | |
| usages.append((candidate, metadata)) | |
| usages.sort(key=lambda x: x[1].relevance_score, reverse=True) | |
| self.cache.set(chunk.chunk_id, cache_type, usages) | |
| return usages | |
| def get_structural_context( | |
| self, | |
| chunk: CodeChunk, | |
| chunks: list[CodeChunk], | |
| ) -> list[tuple[CodeChunk, ContextMetadata]]: | |
| """ | |
| Get structurally related chunks. | |
| This includes parent classes, sibling methods, nested functions, etc. | |
| Args: | |
| chunk: The chunk to analyze | |
| chunks: All available chunks to search | |
| Returns: | |
| List of (chunk, metadata) tuples for structural relations | |
| """ | |
| cache_type = self._candidate_cache_type("structural", chunks) | |
| cached = self.cache.get(chunk.chunk_id, cache_type) | |
| if cached is not None: | |
| return cached | |
| structural_relations = [] | |
| file_chunks = [c for c in chunks if c.file_path == chunk.file_path] | |
| for candidate in file_chunks: | |
| if candidate.chunk_id == chunk.chunk_id: | |
| continue | |
| relevance_score = 0.0 | |
| if self._is_parent_of(candidate, chunk): | |
| relevance_score = 1.0 | |
| elif self._is_child_of(candidate, chunk): | |
| relevance_score = 0.9 | |
| elif self._is_sibling_of(candidate, chunk): | |
| relevance_score = 0.8 | |
| elif self._is_in_same_class(candidate, chunk): | |
| relevance_score = 0.7 | |
| if relevance_score > 0: | |
| metadata = ContextMetadata( | |
| relevance_score=relevance_score, | |
| relationship_type="structural", | |
| distance=abs(candidate.start_line - chunk.start_line), | |
| token_count=len(candidate.content.split()), | |
| ) | |
| structural_relations.append((candidate, metadata)) | |
| structural_relations.sort( | |
| key=lambda x: (x[1].relevance_score, -x[1].distance), | |
| reverse=True, | |
| ) | |
| self.cache.set(chunk.chunk_id, cache_type, structural_relations) | |
| return structural_relations | |
| def _candidate_cache_type(context_type: str, chunks: list[CodeChunk]) -> str: | |
| identities = sorted( | |
| f"{candidate.chunk_id}:{candidate.node_id}:{candidate.file_path}:{candidate.byte_start}" | |
| for candidate in chunks | |
| ) | |
| digest = hashlib.sha256("\0".join(identities).encode("utf-8")).hexdigest()[:16] | |
| return f"{context_type}:{digest}" | |
| _MAX_CONTEXT_CANDIDATES = 200 | |
| def _candidate_subset( | |
| self, | |
| chunk: CodeChunk, | |
| chunk_features: dict[str, Any], | |
| file_chunks: list[CodeChunk], | |
| ) -> list[CodeChunk]: | |
| """Return a BOUNDED candidate set sharing an identifier with the query. | |
| Uses an inverted identifier index (identifier -> chunks) so the similarity | |
| scan considers only candidates that share >= 1 identifier, capped at | |
| _MAX_CONTEXT_CANDIDATES — bounding the former O(n^2) all-pairs pass (COREFIX). | |
| """ | |
| index = self._identifier_index(chunk.file_path, chunk.language, file_chunks) | |
| query_ids = chunk_features.get("identifiers") or set() | |
| seen: dict[Any, Any] = {} | |
| # Iterate identifiers in a STABLE (sorted) order so the cap-truncated | |
| # candidate set is a deterministic function of the input, not of the | |
| # process's hash seed (COREFIX determinism). | |
| for ident in sorted(query_ids): | |
| for cand in index.get(ident, ()): | |
| if cand.chunk_id != chunk.chunk_id: | |
| seen[cand.chunk_id or id(cand)] = cand | |
| if len(seen) >= self._MAX_CONTEXT_CANDIDATES: | |
| return self._ordered(seen.values()) | |
| if seen: | |
| return self._ordered(seen.values()) | |
| return sorted( | |
| (c for c in file_chunks if c.chunk_id != chunk.chunk_id), | |
| key=self._candidate_sort_key, | |
| )[: self._MAX_CONTEXT_CANDIDATES] | |
| def _candidate_sort_key(cand: CodeChunk) -> tuple[str, int]: | |
| return (cand.chunk_id or "", cand.byte_start) | |
| def _ordered(self, candidates: Iterable[CodeChunk]) -> list[CodeChunk]: | |
| return sorted(candidates, key=self._candidate_sort_key) | |
| def _identifier_index( | |
| self, | |
| file_path: str, | |
| language: str, | |
| file_chunks: list[CodeChunk], | |
| ) -> dict[str, list[CodeChunk]]: | |
| # The key includes a CONTENT digest (not just len) so a same-length | |
| # rechunk with different content does not serve a stale index | |
| # (COREFIX cache-freshness). The digest is content-derived via node_id. | |
| digest = self._candidate_cache_type("ident-index", file_chunks) | |
| key = f"{file_path}|{language}|{digest}" | |
| cache = getattr(self, "_ident_index_cache", None) | |
| if cache is None: | |
| cache = {} | |
| self._ident_index_cache = cache | |
| idx = cache.get(key) | |
| if idx is None: | |
| idx = {} | |
| for cand in file_chunks: | |
| feats = self._features_for(cand) | |
| # Index only DISCRIMINATIVE identifiers: drop language keywords and | |
| # ubiquitous short tokens so common words (def/return/self/...) don't | |
| # make every chunk a candidate for every query (keeps the pass bounded). | |
| keywords = feats.get("keywords") or set() | |
| for ident in feats.get("identifiers") or set(): | |
| if ident in keywords or len(ident) <= 2: | |
| continue | |
| idx.setdefault(ident, []).append(cand) | |
| cache[key] = idx | |
| return idx | |
| def _features_for(self, chunk: CodeChunk) -> dict[str, Any]: | |
| """Return cached semantic features for a chunk (computed once per chunk_id).""" | |
| key = chunk.chunk_id or f"{chunk.file_path}:{chunk.byte_start}:{chunk.byte_end}" | |
| cached = self._feature_cache.get(key) | |
| if cached is None: | |
| cached = self._extract_semantic_features(chunk) | |
| self._feature_cache[key] = cached | |
| return cached | |
| def _extract_semantic_features(chunk: CodeChunk) -> dict[str, Any]: | |
| """Extract semantic features from a chunk for similarity comparison.""" | |
| features = { | |
| "identifiers": set(), | |
| "keywords": set(), | |
| "node_type": chunk.node_type, | |
| "parent_context": chunk.parent_context, | |
| "comments": [], | |
| } | |
| identifier_pattern = "\\b[a-zA-Z_][a-zA-Z0-9_]*\\b" | |
| features["identifiers"] = set(re.findall(identifier_pattern, chunk.content)) | |
| if chunk.language == "python": | |
| keywords = { | |
| "def", | |
| "class", | |
| "import", | |
| "from", | |
| "return", | |
| "if", | |
| "else", | |
| "for", | |
| "while", | |
| "try", | |
| "except", | |
| } | |
| elif chunk.language == "javascript": | |
| keywords = { | |
| "function", | |
| "class", | |
| "const", | |
| "let", | |
| "var", | |
| "import", | |
| "export", | |
| "return", | |
| "if", | |
| "else", | |
| } | |
| else: | |
| keywords = set() | |
| content_words = set(chunk.content.lower().split()) | |
| features["keywords"] = keywords.intersection(content_words) | |
| comment_pattern = "#.*?$|//.*?$|/\\*.*?\\*/" | |
| features["comments"] = re.findall( | |
| comment_pattern, | |
| chunk.content, | |
| re.MULTILINE | re.DOTALL, | |
| ) | |
| return features | |
| def _calculate_semantic_similarity( | |
| features1: dict[str, Any], | |
| features2: dict[str, Any], | |
| ) -> float: | |
| """Calculate semantic similarity between two feature sets.""" | |
| score = 0.0 | |
| if features1["node_type"] == features2["node_type"]: | |
| score += 0.3 | |
| if features1["identifiers"] and features2["identifiers"]: | |
| overlap = len( | |
| features1["identifiers"].intersection(features2["identifiers"]), | |
| ) | |
| total = len(features1["identifiers"].union(features2["identifiers"])) | |
| if total > 0: | |
| score += 0.4 * (overlap / total) | |
| if features1["keywords"] and features2["keywords"]: | |
| overlap = len(features1["keywords"].intersection(features2["keywords"])) | |
| total = len(features1["keywords"].union(features2["keywords"])) | |
| if total > 0: | |
| score += 0.2 * (overlap / total) | |
| if features1["parent_context"] == features2["parent_context"]: | |
| score += 0.1 | |
| return min(score, 1.0) | |
| def _extract_imports(chunk: CodeChunk) -> set[str]: | |
| """Extract import statements from a chunk.""" | |
| imports = set() | |
| if chunk.language == "python": | |
| import_pattern = "(?:from\\s+(\\S+)\\s+)?import\\s+([^;\\n]+)" | |
| matches = re.findall(import_pattern, chunk.content) | |
| for module, names in matches: | |
| if module: | |
| imports.add(module) | |
| imports.update(name.strip() for name in names.split(",")) | |
| elif chunk.language in {"javascript", "typescript"}: | |
| import_pattern = ( | |
| "import\\s+(?:{[^}]+}|[^;]+)\\s+from\\s+[\\'\"]([^\\'\"]+)[\\'\"]" | |
| ) | |
| imports.update(re.findall(import_pattern, chunk.content)) | |
| return imports | |
| def _extract_function_calls(chunk: CodeChunk) -> set[str]: | |
| """Extract function call names from a chunk.""" | |
| call_pattern = "(\\b[a-zA-Z_][a-zA-Z0-9_]*)\\s*\\(" | |
| calls = set(re.findall(call_pattern, chunk.content)) | |
| language_keywords = { | |
| "python": {"if", "elif", "while", "for", "except", "with", "def", "class"}, | |
| "javascript": {"if", "while", "for", "function", "catch", "switch"}, | |
| } | |
| keywords = language_keywords.get(chunk.language, set()) | |
| return calls - keywords | |
| def _extract_class_references(chunk: CodeChunk) -> set[str]: | |
| """Extract class references from a chunk.""" | |
| references = set() | |
| if chunk.language == "python": | |
| class_pattern = "class\\s+\\w+\\s*\\(([^)]+)\\)" | |
| matches = re.findall(class_pattern, chunk.content) | |
| for match in matches: | |
| references.update(base.strip() for base in match.split(",")) | |
| elif chunk.language in {"javascript", "typescript"}: | |
| extend_pattern = "class\\s+\\w+\\s+extends\\s+(\\w+)" | |
| references.update(re.findall(extend_pattern, chunk.content)) | |
| return references | |
| def _defines_import(self, chunk: CodeChunk, imports: set[str]) -> bool: | |
| """Check if a chunk defines any of the imported items.""" | |
| defined = self._extract_exports(chunk) | |
| return bool( | |
| imports.intersection( | |
| defined["functions"] | defined["classes"] | defined["variables"], | |
| ), | |
| ) | |
| def _defines_function( | |
| self, | |
| chunk: CodeChunk, | |
| function_calls: set[str], | |
| ) -> bool: | |
| """Check if a chunk defines any of the called functions.""" | |
| defined = self._extract_exports(chunk) | |
| return bool(function_calls.intersection(defined["functions"])) | |
| def _defines_class( | |
| self, | |
| chunk: CodeChunk, | |
| class_references: set[str], | |
| ) -> bool: | |
| """Check if a chunk defines any of the referenced classes.""" | |
| defined = self._extract_exports(chunk) | |
| return bool(class_references.intersection(defined["classes"])) | |
| def _extract_exports(chunk: CodeChunk) -> dict[str, set[str]]: | |
| """Extract what a chunk exports/defines.""" | |
| exports = {"functions": set(), "classes": set(), "variables": set()} | |
| if chunk.language == "python": | |
| func_pattern = "def\\s+([a-zA-Z_][a-zA-Z0-9_]*)" | |
| exports["functions"].update( | |
| re.findall(func_pattern, chunk.content), | |
| ) | |
| class_pattern = "class\\s+([a-zA-Z_][a-zA-Z0-9_]*)" | |
| exports["classes"].update(re.findall(class_pattern, chunk.content)) | |
| var_pattern = "^([a-zA-Z_][a-zA-Z0-9_]*)\\s*=" | |
| exports["variables"].update( | |
| re.findall(var_pattern, chunk.content, re.MULTILINE), | |
| ) | |
| elif chunk.language in {"javascript", "typescript"}: | |
| func_pattern = r"function\s+([a-zA-Z_][a-zA-Z0-9_]*)" | |
| exports["functions"].update( | |
| re.findall(func_pattern, chunk.content), | |
| ) | |
| # Arrow function declarations: const Name = (...) => { ... } | |
| arrow_pattern = ( | |
| r"(?:export\\s+)?(?:const|let|var)\\s+([a-zA-Z_][a-zA-Z0-9_]*)\\s*=\\s*" | |
| r"(?:\\([^)]*\\)|[a-zA-Z_][a-zA-Z0-9_]*|\\(.*?\\))\\s*=>" | |
| ) | |
| exports["functions"].update(re.findall(arrow_pattern, chunk.content)) | |
| # React component declarations: function Name(...) { ... } | |
| react_fn_component = r"(?:export\s+)?function\s+([A-Z][A-Za-z0-9_]*)\s*\(" | |
| exports["functions"].update(re.findall(react_fn_component, chunk.content)) | |
| # React component as class | |
| class_pattern = "class\\s+([A-Z][A-Za-z0-9_]*)" | |
| exports["classes"].update(re.findall(class_pattern, chunk.content)) | |
| return exports | |
| def _imports_from( | |
| self, | |
| candidate: CodeChunk, | |
| source_chunk: CodeChunk, | |
| exports: dict[str, set[str]], | |
| ) -> bool: | |
| """Check if candidate imports from source chunk.""" | |
| imports = self._extract_imports(candidate) | |
| source_file_name = ( | |
| source_chunk.file_path.split("/")[-1].replace(".py", "").replace(".js", "") | |
| ) | |
| if source_file_name in imports: | |
| return True | |
| all_exports = exports["functions"] | exports["classes"] | exports["variables"] | |
| return bool(imports.intersection(all_exports)) | |
| def _calls_functions_from( | |
| self, | |
| candidate: CodeChunk, | |
| exports: dict[str, set[str]], | |
| ) -> bool: | |
| """Check if candidate calls functions from exports.""" | |
| calls = self._extract_function_calls(candidate) | |
| return bool(calls.intersection(exports["functions"])) | |
| def _uses_classes_from( | |
| self, | |
| candidate: CodeChunk, | |
| exports: dict[str, set[str]], | |
| ) -> bool: | |
| """Check if candidate uses classes from exports.""" | |
| references = self._extract_class_references(candidate) | |
| return bool(references.intersection(exports["classes"])) | |
| def _is_parent_of(candidate: CodeChunk, chunk: CodeChunk) -> bool: | |
| """Check if candidate is a parent of chunk.""" | |
| return ( | |
| candidate.start_line <= chunk.start_line | |
| and candidate.end_line >= chunk.end_line | |
| and candidate.chunk_id != chunk.chunk_id | |
| ) | |
| def _is_child_of(candidate: CodeChunk, chunk: CodeChunk) -> bool: | |
| """Check if candidate is a child of chunk.""" | |
| return ( | |
| chunk.start_line <= candidate.start_line | |
| and chunk.end_line >= candidate.end_line | |
| and candidate.chunk_id != chunk.chunk_id | |
| ) | |
| def _is_sibling_of(self, candidate: CodeChunk, chunk: CodeChunk) -> bool: | |
| """Check if candidate is a sibling of chunk.""" | |
| return ( | |
| candidate.parent_context == chunk.parent_context | |
| and candidate.chunk_id != chunk.chunk_id | |
| and not self._is_parent_of(candidate, chunk) | |
| and not self._is_child_of(candidate, chunk) | |
| ) | |
| def _is_in_same_class(candidate: CodeChunk, chunk: CodeChunk) -> bool: | |
| """Check if candidate is in the same class as chunk.""" | |
| if not chunk.parent_context or not candidate.parent_context: | |
| return False | |
| class_pattern = "class\\s+([a-zA-Z_][a-zA-Z0-9_]*)" | |
| chunk_classes = set(re.findall(class_pattern, chunk.parent_context)) | |
| candidate_classes = set(re.findall(class_pattern, candidate.parent_context)) | |
| return bool(chunk_classes.intersection(candidate_classes)) | |
| def _calculate_file_distance(chunk1: CodeChunk, chunk2: CodeChunk) -> int: | |
| """Calculate distance between chunks across files.""" | |
| if chunk1.file_path == chunk2.file_path: | |
| return abs(chunk1.start_line - chunk2.start_line) | |
| return 10000 | |
| def _get_file_chunks(file_path: str, language: str) -> list[CodeChunk]: | |
| """Get all chunks from a file using the core chunker. | |
| Falls back to empty list on any parsing error to avoid breaking context. | |
| """ | |
| try: | |
| from .core import chunk_file as core_chunk_file | |
| return core_chunk_file(file_path, language) | |
| except Exception: | |
| return [] | |
| def _get_parser(self, language: str): | |
| """Return the calling thread's parser for the language. | |
| Delegates to the thread-local ``get_parser`` (IF-0-PARSER-1) on every | |
| call rather than caching one Parser in a shared instance dict — a | |
| SmartContextProvider shared across threads must never hand one Parser to | |
| two threads (the shared-parser segfault class SCALE closes). | |
| """ | |
| return get_parser(language) | |
| class RelevanceContextStrategy(ContextStrategy): | |
| """ | |
| Strategy for selecting context based on relevance scores. | |
| This strategy prioritizes chunks with the highest relevance scores | |
| while respecting token limits. | |
| """ | |
| def select_context( | |
| _chunk: CodeChunk, | |
| candidates: list[tuple[CodeChunk, ContextMetadata]], | |
| max_tokens: int, | |
| ) -> list[CodeChunk]: | |
| """ | |
| Select the most relevant context chunks. | |
| Args: | |
| chunk: The main chunk | |
| candidates: List of (chunk, metadata) tuples to select from | |
| max_tokens: Maximum tokens to include | |
| Returns: | |
| Selected chunks ordered by relevance | |
| """ | |
| sorted_candidates = sorted( | |
| candidates, | |
| key=lambda x: x[1].relevance_score, | |
| reverse=True, | |
| ) | |
| selected = [] | |
| total_tokens = 0 | |
| for candidate_chunk, metadata in sorted_candidates: | |
| if total_tokens + metadata.token_count <= max_tokens: | |
| selected.append(candidate_chunk) | |
| total_tokens += metadata.token_count | |
| elif total_tokens < max_tokens: | |
| remaining_tokens = max_tokens - total_tokens | |
| if remaining_tokens > 100: | |
| selected.append(candidate_chunk) | |
| break | |
| return selected | |
| def rank_candidates( | |
| _chunk: CodeChunk, | |
| candidates: list[tuple[CodeChunk, ContextMetadata]], | |
| ) -> list[tuple[CodeChunk, float]]: | |
| """ | |
| Rank candidate chunks by relevance. | |
| Args: | |
| chunk: The main chunk | |
| candidates: List of (chunk, metadata) tuples | |
| Returns: | |
| List of (chunk, score) tuples sorted by score descending | |
| """ | |
| ranked = [] | |
| for candidate_chunk, metadata in candidates: | |
| score = metadata.relevance_score | |
| if metadata.relationship_type == "dependency": | |
| score *= 1.2 | |
| elif metadata.relationship_type == "structural": | |
| score *= 1.1 | |
| if metadata.distance > 0: | |
| distance_penalty = 1.0 / (1.0 + metadata.distance / 100.0) | |
| score *= distance_penalty | |
| ranked.append((candidate_chunk, score)) | |
| ranked.sort(key=lambda x: x[1], reverse=True) | |
| return ranked | |
| class HybridContextStrategy(ContextStrategy): | |
| """ | |
| Hybrid strategy that balances different types of context. | |
| This strategy ensures a mix of dependency, usage, semantic, and structural | |
| context for comprehensive understanding. | |
| """ | |
| def __init__(self, weights: dict[str, float] | None = None): | |
| """ | |
| Initialize the hybrid strategy. | |
| Args: | |
| weights: Optional weights for different relationship types | |
| """ | |
| self.weights = weights or { | |
| "dependency": 0.35, | |
| "usage": 0.25, | |
| "semantic": 0.25, | |
| "structural": 0.15, | |
| } | |
| def select_context( | |
| self, | |
| _chunk: CodeChunk, | |
| candidates: list[tuple[CodeChunk, ContextMetadata]], | |
| max_tokens: int, | |
| ) -> list[CodeChunk]: | |
| """ | |
| Select a balanced mix of context chunks. | |
| Args: | |
| chunk: The main chunk | |
| candidates: List of (chunk, metadata) tuples to select from | |
| max_tokens: Maximum tokens to include | |
| Returns: | |
| Selected chunks ordered by relevance | |
| """ | |
| grouped = defaultdict(list) | |
| for candidate_chunk, metadata in candidates: | |
| grouped[metadata.relationship_type].append((candidate_chunk, metadata)) | |
| type_budgets = {} | |
| for rel_type, weight in self.weights.items(): | |
| type_budgets[rel_type] = int(max_tokens * weight) | |
| selected = [] | |
| for rel_type, budget in type_budgets.items(): | |
| if rel_type not in grouped: | |
| continue | |
| group_sorted = sorted( | |
| grouped[rel_type], | |
| key=lambda x: x[1].relevance_score, | |
| reverse=True, | |
| ) | |
| group_tokens = 0 | |
| for candidate_chunk, metadata in group_sorted: | |
| if group_tokens + metadata.token_count <= budget: | |
| selected.append(candidate_chunk) | |
| group_tokens += metadata.token_count | |
| return selected | |
| def rank_candidates( | |
| self, | |
| _chunk: CodeChunk, | |
| candidates: list[tuple[CodeChunk, ContextMetadata]], | |
| ) -> list[tuple[CodeChunk, float]]: | |
| """ | |
| Rank candidate chunks by weighted relevance. | |
| Args: | |
| chunk: The main chunk | |
| candidates: List of (chunk, metadata) tuples | |
| Returns: | |
| List of (chunk, score) tuples sorted by score descending | |
| """ | |
| ranked = [] | |
| for candidate_chunk, metadata in candidates: | |
| weight = self.weights.get(metadata.relationship_type, 0.1) | |
| score = metadata.relevance_score * weight | |
| if metadata.distance < 50: | |
| score *= 1.1 | |
| ranked.append((candidate_chunk, score)) | |
| ranked.sort(key=lambda x: x[1], reverse=True) | |
| return ranked | |
| class InMemoryContextCache(ContextCache): | |
| """In-memory cache implementation for context computations.""" | |
| def __init__(self, ttl: int = 3600): | |
| """ | |
| Initialize the cache. | |
| Args: | |
| ttl: Time-to-live in seconds for cache entries | |
| """ | |
| self.cache: dict[ | |
| str, | |
| dict[str, tuple[list[tuple[CodeChunk, ContextMetadata]], float]], | |
| ] = {} | |
| self.ttl = ttl | |
| # Back-compat helper for tests | |
| def size(self) -> int: | |
| return len(self.cache) | |
| def get( | |
| self, | |
| chunk_id: str, | |
| context_type: str, | |
| ) -> list[tuple[CodeChunk, ContextMetadata]] | None: | |
| """Get cached context if available and not expired.""" | |
| cache_key = f"{chunk_id}_{context_type}" | |
| if cache_key in self.cache: | |
| data, timestamp = self.cache[cache_key] | |
| if time.time() - timestamp < self.ttl: | |
| return data | |
| del self.cache[cache_key] | |
| return None | |
| def set( | |
| self, | |
| chunk_id: str, | |
| context_type: str, | |
| context: list[tuple[CodeChunk, ContextMetadata]], | |
| ) -> None: | |
| """Cache context for a chunk.""" | |
| cache_key = f"{chunk_id}_{context_type}" | |
| self.cache[cache_key] = context, time.time() | |
| def invalidate(self, chunk_ids: builtins.set[str] | None = None) -> None: | |
| """Invalidate cache entries.""" | |
| if chunk_ids is None: | |
| self.cache.clear() | |
| else: | |
| keys_to_remove = [] | |
| for key in self.cache: | |
| chunk_id = key.split("_")[0] | |
| if chunk_id in chunk_ids: | |
| keys_to_remove.append(key) | |
| for key in keys_to_remove: | |
| del self.cache[key] | |
Xet Storage Details
- Size:
- 34.1 kB
- Xet hash:
- 5712b8b9a853fd7efd94b6d79ef257a679d54537a273de63740e78090182cb94
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.