Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /multi_language.py
"""Multi-language project processing implementation."""
import json
import os
import re
from collections import defaultdict
from pathlib import Path
from typing import Any, ClassVar
from .core import (
chunk_file,
chunk_text,
) # noqa: F401 (chunk_file kept for test patch target)
from .interfaces.multi_language import (
CrossLanguageReference,
EmbeddedLanguageType,
LanguageDetector,
LanguageRegion,
MultiLanguageProcessor,
ProjectAnalyzer,
)
from .parser import get_parser, list_languages
from .types import CodeChunk, compute_node_id
try:
pass
except ImportError:
def list_languages() -> list[str]:
return ["python", "javascript", "typescript", "java", "go", "rust", "c", "cpp"]
def get_parser(_language: Any, _config: Any = None) -> Any: # type: ignore[misc]
raise ImportError("Tree-sitter parser not available")
def chunk_file(_path: Any, _language: Any = None, **_kwargs: Any) -> Any: # type: ignore[misc]
raise ImportError("Chunker not available")
def chunk_text(_text: Any, _language: Any = None, **_kwargs: Any) -> Any: # type: ignore[misc]
raise ImportError("Chunker not available")
class LanguageDetectorImpl(LanguageDetector):
"""Detect programming languages in files and content."""
EXTENSIONS: ClassVar[dict[str, str]] = {
".py": "python",
".js": "javascript",
".jsx": "javascript",
".ts": "typescript",
".tsx": "typescript",
".java": "java",
".cpp": "cpp",
".cc": "cpp",
".cxx": "cpp",
".c": "c",
".h": "c",
".hpp": "cpp",
".rs": "rust",
".go": "go",
".rb": "ruby",
".php": "php",
".swift": "swift",
".kt": "kotlin",
".scala": "scala",
".r": "r",
".m": "objc",
".mm": "objc",
".cs": "csharp",
".vb": "vb",
".fs": "fsharp",
".ml": "ocaml",
".lua": "lua",
".pl": "perl",
".sh": "bash",
".bash": "bash",
".zsh": "bash",
".fish": "bash",
".ps1": "powershell",
".psm1": "powershell",
".html": "html",
".htm": "html",
".xml": "xml",
".css": "css",
".scss": "scss",
".sass": "sass",
".less": "less",
".json": "json",
".yaml": "yaml",
".yml": "yaml",
".toml": "toml",
".ini": "ini",
".cfg": "ini",
".md": "markdown",
".markdown": "markdown",
".rst": "rst",
".tex": "latex",
".sql": "sql",
".graphql": "graphql",
".gql": "graphql",
".ipynb": "jupyter",
}
SHEBANG_PATTERNS: ClassVar[dict[str, str]] = {
"python[0-9\\.]*": "python",
"node": "javascript",
"ruby": "ruby",
"perl": "perl",
"bash": "bash",
"sh": "bash",
"zsh": "bash",
"fish": "bash",
"lua": "lua",
"php": "php",
}
CONTENT_PATTERNS: ClassVar[dict[str, list[str]]] = {
"python": [
"^\\s*import\\s+\\w+(?:\\s*,\\s*\\w+)*\\s*$",
"^\\s*from\\s+[\\w\\.]+\\s+import",
"^\\s*def\\s+\\w+\\s*\\(",
"^\\s*class\\s+\\w+\\s*[:\\(]",
"^\\s*if\\s+__name__\\s*==\\s*[\"\\']__main__[\"\\']",
"^\\s*@\\w+",
],
"javascript": [
"^\\s*const\\s+\\w+\\s*=",
"^\\s*let\\s+\\w+\\s*=",
"^\\s*var\\s+\\w+\\s*=",
"^\\s*function\\s+\\w+\\s*\\(",
"^\\s*class\\s+\\w+\\s*[\\{]",
"^\\s*import\\s+.*\\s+from\\s+[\"\\']",
"^\\s*export\\s+(default\\s+)?",
],
"typescript": [
"^\\s*interface\\s+\\w+\\s*[\\{]",
"^\\s*type\\s+\\w+\\s*=",
"^\\s*enum\\s+\\w+\\s*[\\{]",
":\\s*(string|number|boolean|any|void|never|unknown)\\s*[;,\\)\\}]",
],
"java": [
"^\\s*package\\s+[\\w\\.]+;",
"^\\s*import\\s+[\\w\\.]+\\.*;?\\s*$",
"^\\s*public\\s+class\\s+\\w+",
"^\\s*private\\s+\\w+\\s+\\w+;",
"^\\s*public\\s+static\\s+void\\s+main",
"^\\s*(public|private|protected)\\s+\\w+\\s+\\w+\\s*[;=\\(]",
],
"go": [
"^\\s*package\\s+\\w+",
"^\\s*import\\s+\\(",
"^\\s*func\\s+\\w+\\s*\\(",
"^\\s*type\\s+\\w+\\s+struct\\s*\\{",
"^\\s*var\\s+\\w+\\s+\\w+",
],
"rust": [
"^\\s*use\\s+\\w+",
"^\\s*fn\\s+\\w+\\s*\\(",
"^\\s*struct\\s+\\w+\\s*[\\{\\(]",
"^\\s*impl\\s+\\w+",
"^\\s*let\\s+(mut\\s+)?\\w+",
"^\\s*pub\\s+(fn|struct|enum|trait)",
],
"ruby": [
"^\\s*require\\s+[\"\\']",
"^\\s*require_relative\\s+[\"\\']",
"^\\s*def\\s+\\w+",
"^\\s*class\\s+\\w+",
"^\\s*module\\s+\\w+",
"^\\s*attr_(reader|writer|accessor)\\s+",
],
"php": [
"<\\?php",
"^\\s*namespace\\s+[\\w\\\\\\\\]+;",
"^\\s*use\\s+[\\w\\\\\\\\]+;",
"^\\s*class\\s+\\w+",
"^\\s*function\\s+\\w+\\s*\\(",
"\\$\\w+\\s*=",
],
}
def detect_from_file(self, file_path: str) -> tuple[str, float]: # type: ignore[override]
"""Detect language from file_path path and content."""
path = Path(file_path)
confidence = 0.0
language = None
ext = path.suffix.lower()
if ext in self.EXTENSIONS:
language = self.EXTENSIONS[ext]
confidence = 0.8
try:
with Path(file_path).open(encoding="utf-8", errors="ignore") as f:
content = f.read(4096)
if content.startswith("#!"):
first_line = content.split("\n")[0]
for pattern, lang in self.SHEBANG_PATTERNS.items():
if re.search(pattern, first_line):
return lang, 0.95
if language:
content_lang, content_conf = self.detect_from_content(
content,
hint=language,
)
if content_lang == language:
confidence = min(0.95, confidence + content_conf * 0.2)
elif content_conf > 0.8:
language = content_lang
confidence = content_conf
else:
language, confidence = self.detect_from_content(content)
except OSError:
pass
if not language:
language = "text"
confidence = 0.1
return language, confidence
def detect_from_content( # type: ignore[override]
self,
content: str,
hint: str | None = None,
) -> tuple[str, float]:
"""Detect language from content alone."""
if not content.strip():
return "text", 0.1
scores = defaultdict(float)
if hint and hint in self.CONTENT_PATTERNS:
scores[hint] = 0.2
for language, patterns in self.CONTENT_PATTERNS.items():
for pattern in patterns:
matches = len(re.findall(pattern, content, re.MULTILINE))
if matches:
scores[language] += matches * 0.1
if "typescript" in scores and "javascript" in scores:
scores["typescript"] += scores["javascript"] * 0.5
if scores:
max_score = max(scores.values())
if max_score > 0:
for lang in scores:
scores[lang] = min(0.95, scores[lang] / max_score)
best_lang = max(scores.items(), key=lambda x: x[1])
return best_lang
return "text", 0.1
def detect_multiple(self, content: str) -> list[tuple[str, float]]: # type: ignore[override]
"""Detect multiple languages in content."""
if not content.strip():
return [("text", 1.0)]
language_blocks = []
# Detect fenced code blocks and include both language and content size
markdown_blocks = list(
re.finditer(r"```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```", content),
)
for m in markdown_blocks:
lang = m.group(1) or None
block = m.group(2)
if lang:
language_blocks.append((lang.lower(), len(block)))
else:
detected_lang, _ = self.detect_from_content(block)
language_blocks.append((detected_lang, len(block)))
script_blocks = re.findall(
r"<script[^>]*>(.*?)</script>",
content,
re.DOTALL | re.IGNORECASE,
)
language_blocks.extend(("javascript", len(block)) for block in script_blocks)
style_blocks = re.findall(
r"<style[^>]*>(.*?)</style>",
content,
re.DOTALL | re.IGNORECASE,
)
language_blocks.extend(("css", len(block)) for block in style_blocks)
if re.search(r"<[A-Z]\\w*[^>]*>", content) and re.search(
r"(import|export|const|let|var)",
content,
):
ts_patterns = len(
re.findall(
r":\\s*(string|number|boolean|any|void)\\s*[;,\\)\\}]",
content,
),
)
if ts_patterns > 2:
language_blocks.append(("typescript", len(content)))
else:
language_blocks.append(("javascript", len(content)))
if language_blocks:
total_size = sum(size for _, size in language_blocks)
language_percentages: dict[str, float] = defaultdict(float)
for lang, size in language_blocks:
language_percentages[lang] += size / total_size
results = sorted(
language_percentages.items(),
key=lambda x: x[1],
reverse=True,
)
return results
lang, _conf = self.detect_from_content(content)
return [(lang, 1.0)]
class ProjectAnalyzerImpl(ProjectAnalyzer):
"""Analyze multi-language project structure."""
def __init__(self, detector: LanguageDetector | None = None):
self.detector = detector or LanguageDetectorImpl()
def analyze_structure(self, project_path: str) -> dict[str, Any]: # type: ignore[override]
"""Analyze overall project structure."""
project_root = Path(project_path)
if not project_root.exists():
raise ValueError(f"Project path does not exist: {project_path}")
analysis: dict[str, Any] = {
"project_path": str(project_root),
"languages": defaultdict(int),
"file_count": 0,
"total_lines": 0,
"framework_indicators": {},
"project_type": "unknown",
"structure": {
"has_backend": False,
"has_frontend": False,
"has_tests": False,
"has_docs": False,
"has_config": False,
},
}
framework_files = {
"package.json": ["javascript", "node", "npm"],
"tsconfig.json": ["typescript"],
"requirements.txt": ["python"],
"setup.py": ["python"],
"pyproject.toml": ["python"],
"Cargo.toml": ["rust"],
"go.mod": ["go"],
"pom.xml": ["java", "maven"],
"build.gradle": ["java", "gradle"],
"Gemfile": ["ruby"],
"composer.json": ["php"],
"CMakeLists.txt": ["cpp", "cmake"],
"Makefile": ["make"],
"Dockerfile": ["docker"],
"docker-compose.yml": ["docker"],
".gitignore": ["git"],
}
for root, dirs, files in os.walk(project_root):
dirs[:] = [
d
for d in dirs
if not d.startswith(".")
and d
not in {
"node_modules",
"venv",
"env",
"__pycache__",
"target",
"build",
"dist",
}
]
rel_path = Path(root).relative_to(project_root)
for filename in files:
file_path = Path(root) / filename
analysis["file_count"] += 1
if filename in framework_files:
for indicator in framework_files[filename]:
analysis["framework_indicators"][indicator] = True
# Heuristic: presence of JS build files implies frontend
if any(
k in framework_files[filename]
for k in ["javascript", "node", "npm", "typescript"]
):
analysis["structure"]["has_frontend"] = True
try:
lang, confidence = self.detector.detect_from_file(str(file_path))
if confidence > 0.5:
analysis["languages"][lang] += 1
try:
with Path(file_path).open(
encoding="utf-8",
errors="ignore",
) as f:
analysis["total_lines"] += sum(1 for _ in f)
except (OSError, FileNotFoundError, IndexError):
pass
path_parts = str(rel_path).lower()
if any(
part in path_parts
for part in ["backend", "server", "api", "src/main"]
):
analysis["structure"]["has_backend"] = True
if any(
part in path_parts
for part in [
"frontend",
"client",
"web",
"static",
"public",
"src/components",
]
):
analysis["structure"]["has_frontend"] = True
if any(
part in path_parts
for part in ["test", "tests", "spec", "__tests__"]
):
analysis["structure"]["has_tests"] = True
if any(
part in path_parts
for part in ["docs", "documentation", "README"]
):
analysis["structure"]["has_docs"] = True
if filename in framework_files or str(file_path).endswith(
(".json", ".yaml", ".yml", ".toml", ".ini"),
):
analysis["structure"]["has_config"] = True
except (OSError, FileNotFoundError, IndexError):
pass
analysis["project_type"] = self._determine_project_type(analysis)
analysis["languages"] = dict(analysis["languages"])
return analysis
@staticmethod
def _determine_project_type(analysis: dict[str, Any]) -> str:
"""Determine project type from analysis."""
indicators = analysis["framework_indicators"]
structure = analysis["structure"]
languages = analysis["languages"]
# Check project types in priority order
from typing import Callable
type_checks: list[tuple[Callable[[], bool], str]] = [
# Fullstack first
(
lambda: structure["has_frontend"] and structure["has_backend"],
"fullstack_webapp",
),
# Prefer backend classification before frontend-only and node app
(
lambda: structure["has_backend"] and not structure["has_frontend"],
"backend_api",
),
(
lambda: structure["has_frontend"]
and ProjectAnalyzerImpl._is_node_app(indicators),
"node_application",
),
(lambda: structure["has_frontend"], "frontend_webapp"),
(
lambda: ProjectAnalyzerImpl._is_library(indicators, analysis),
"library",
),
(
lambda: ProjectAnalyzerImpl._is_mobile_app(languages, analysis),
"mobile_app",
),
(
lambda: "python" in languages and "jupyter" in languages,
"data_science_project",
),
]
for check_func, project_type in type_checks:
if check_func():
return project_type
return "general_project"
@staticmethod
def _is_node_app(indicators: dict[str, Any]) -> bool:
"""Check if project is a Node application."""
return "javascript" in indicators or (
"typescript" in indicators and "node" in indicators
)
@staticmethod
def _is_library(indicators: dict[str, Any], analysis: dict[str, Any]) -> bool:
"""Check if project is a library."""
return (
any(key in indicators for key in ["npm", "python", "rust", "go"])
and analysis["file_count"] < 50
)
@staticmethod
def _is_mobile_app(languages: dict[str, Any], analysis: dict[str, Any]) -> bool:
"""Check if project is a mobile app."""
has_mobile_lang = (
"swift" in languages or "kotlin" in languages or "java" in languages
)
project_path_str = str(analysis["project_path"]).lower()
has_mobile_path = "android" in project_path_str or "ios" in project_path_str
return has_mobile_lang and has_mobile_path
@staticmethod
def find_api_boundaries(chunks: list[CodeChunk]) -> list[dict[str, Any]]:
"""Find API boundaries between components."""
api_boundaries = []
backend_chunks = []
frontend_chunks = []
api_chunks = []
for chunk in chunks:
path_lower = chunk.file_path.lower()
if any(
pattern in path_lower
for pattern in [
"api/",
"/api/",
"routes/",
"controllers/",
"endpoints/",
]
):
api_chunks.append(chunk)
elif any(
pattern in path_lower
for pattern in ["backend/", "server/", "src/main/"]
):
backend_chunks.append(chunk)
elif any(
pattern in path_lower
for pattern in ["frontend/", "client/", "src/components/", "pages/"]
):
frontend_chunks.append(chunk)
if chunk.language in {"python", "javascript", "typescript", "java", "go"}:
rest_patterns = [
"@(app|router)\\.(get|post|put|delete|patch|route)\\(",
"@(Get|Post|Put|Delete|Patch)Mapping",
"router\\.(get|post|put|delete|patch)\\(",
"http\\.(Get|Post|Put|Delete|Patch)\\(",
]
for pattern in rest_patterns:
if re.search(pattern, chunk.content):
endpoint_match = re.search(
r"[\"']([/\w\-\{\}:]+)[\"']",
chunk.content,
)
if endpoint_match:
api_boundaries.append(
{
"type": "rest_endpoint",
"chunk_id": chunk.chunk_id,
"endpoint": endpoint_match.group(1),
"method": "detected",
"language": chunk.language,
"file_path": chunk.file_path,
},
)
graphql_patterns = [
"type\\s+Query\\s*\\{",
"type\\s+Mutation\\s*\\{",
"@(Query|Mutation|Resolver)",
]
api_boundaries.extend(
{
"type": "graphql_schema",
"chunk_id": chunk.chunk_id,
"language": chunk.language,
"file_path": chunk.file_path,
}
for pattern in graphql_patterns
if re.search(pattern, chunk.content)
)
api_boundaries.extend(
{
"type": "grpc_service",
"chunk_id": chunk.chunk_id,
"file_path": chunk.file_path,
}
for chunk in chunks
if chunk.language in {"proto", "protobuf"} or ".proto" in chunk.file_path
)
return api_boundaries
@classmethod
def suggest_chunk_grouping(
cls,
chunks: list[CodeChunk],
) -> dict[str, list[CodeChunk]]:
"""Suggest how to group chunks for processing."""
groupings = defaultdict(list)
for chunk in chunks:
path_parts = Path(chunk.file_path).parts
feature = None
for i, part in enumerate(path_parts):
if part in {
"features",
"modules",
"components",
"services",
"domains",
} and i + 1 < len(path_parts):
feature = path_parts[i + 1]
break
if feature:
groupings[f"feature_{feature}"].append(chunk)
elif len(path_parts) > 1:
groupings[f"module_{path_parts[0]}"].append(chunk)
else:
groupings["root"].append(chunk)
for chunk in chunks:
groupings[f"lang_{chunk.language}"].append(chunk)
for chunk in chunks:
groupings[f"type_{chunk.node_type}"].append(chunk)
return dict(groupings)
class MultiLanguageProcessorImpl(MultiLanguageProcessor):
"""Process projects with multiple languages."""
def __init__(
self,
detector: LanguageDetector | None = None,
analyzer: ProjectAnalyzer | None = None,
):
self.detector = detector or LanguageDetectorImpl()
self.analyzer = analyzer or ProjectAnalyzerImpl(self.detector)
try:
self._supported_languages = set(list_languages())
except (TypeError, ValueError):
self._supported_languages = {
"python",
"javascript",
"typescript",
"java",
"go",
"rust",
"c",
"cpp",
"ruby",
"php",
"swift",
"kotlin",
"csharp",
}
def detect_project_languages(self, project_path: str) -> dict[str, float]: # type: ignore[override]
"""Detect languages used in project with confidence scores."""
analysis = self.analyzer.analyze_structure(project_path)
total_files = sum(analysis["languages"].values())
if total_files == 0:
return {}
language_percentages = {}
for lang, count in analysis["languages"].items():
percentage = count / total_files
language_percentages[lang] = percentage
return language_percentages
def identify_language_regions( # type: ignore[override]
self,
file_path: str,
content: str,
) -> list[LanguageRegion]:
"""Identify regions of different languages within a file_path."""
regions = []
lines = content.split("\n")
primary_lang, _ = self.detector.detect_from_file(file_path)
if file_path.endswith((".jsx", ".tsx")):
regions.extend(self._identify_jsx_regions(content, primary_lang))
elif file_path.endswith((".html", ".htm")):
regions.extend(self._identify_html_regions(content))
elif file_path.endswith(".md"):
regions.extend(self._identify_markdown_regions(content))
elif file_path.endswith(".ipynb"):
regions.extend(self._identify_notebook_regions(content))
else:
regions.extend(self._identify_embedded_regions(content, primary_lang))
if not regions and content.strip():
regions.append(
LanguageRegion(
language=primary_lang,
start_pos=0,
end_pos=len(content),
start_line=1,
end_line=len(lines),
embedding_type=None,
parent_language=None,
),
)
return regions
@classmethod
def _identify_jsx_regions(
cls,
content: str,
base_language: str,
) -> list[LanguageRegion]:
"""Identify JSX/TSX regions."""
regions = []
lines = content.split("\n")
regions.append(
LanguageRegion(
language=base_language,
start_pos=0,
end_pos=len(content),
start_line=1,
end_line=len(lines),
embedding_type=EmbeddedLanguageType.TEMPLATE,
parent_language=None,
),
)
style_pattern = r"style\s*=\s*\{\{([\s\S]*?)\}\}"
for match in re.finditer(style_pattern, content):
start_line = content[: match.start()].count("\n") + 1
end_line = content[: match.end()].count("\n") + 1
regions.append(
LanguageRegion(
language="css",
start_pos=match.start(1),
end_pos=match.end(1),
start_line=start_line,
end_line=end_line,
embedding_type=EmbeddedLanguageType.STYLE,
parent_language=base_language,
),
)
return regions
@classmethod
def _identify_html_regions(cls, content: str) -> list[LanguageRegion]:
"""Identify regions in HTML files."""
regions = []
regions.append(
LanguageRegion(
language="html",
start_pos=0,
end_pos=len(content),
start_line=1,
end_line=content.count("\n") + 1,
embedding_type=None,
parent_language=None,
),
)
script_pattern = "<script[^>]*>(.*?)</script>"
for match in re.finditer(script_pattern, content, re.DOTALL | re.IGNORECASE):
start_line = content[: match.start(1)].count("\n") + 1
end_line = content[: match.end(1)].count("\n") + 1
regions.append(
LanguageRegion(
language="javascript",
start_pos=match.start(1),
end_pos=match.end(1),
start_line=start_line,
end_line=end_line,
embedding_type=EmbeddedLanguageType.SCRIPT,
parent_language="html",
),
)
style_pattern = "<style[^>]*>(.*?)</style>"
for match in re.finditer(style_pattern, content, re.DOTALL | re.IGNORECASE):
start_line = content[: match.start(1)].count("\n") + 1
end_line = content[: match.end(1)].count("\n") + 1
regions.append(
LanguageRegion(
language="css",
start_pos=match.start(1),
end_pos=match.end(1),
start_line=start_line,
end_line=end_line,
embedding_type=EmbeddedLanguageType.STYLE,
parent_language="html",
),
)
return regions
@classmethod
def _identify_markdown_regions(cls, content: str) -> list[LanguageRegion]:
"""Identify regions in Markdown files."""
regions = []
regions.append(
LanguageRegion(
language="markdown",
start_pos=0,
end_pos=len(content),
start_line=1,
end_line=content.count(
"\n",
)
+ 1,
embedding_type=None,
parent_language=None,
),
)
code_block_pattern = r"```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```"
for match in re.finditer(code_block_pattern, content):
language = (match.group(1) or "text").lower()
start_line = content[: match.start(2)].count("\n") + 1
end_line = content[: match.end(2)].count("\n") + 1
regions.append(
LanguageRegion(
language=language,
start_pos=match.start(2),
end_pos=match.end(2),
start_line=start_line,
end_line=end_line,
embedding_type=EmbeddedLanguageType.DOCUMENTATION,
parent_language="markdown",
),
)
return regions
@classmethod
def _identify_notebook_regions(cls, content: str) -> list[LanguageRegion]:
"""Identify regions in Jupyter notebooks."""
regions = []
try:
notebook = json.loads(content)
current_pos = 0
current_line = 1
for cell in notebook.get("cells", []):
cell_type = cell.get("cell_type", "code")
source = cell.get("source", [])
if isinstance(source, list):
source = "".join(source)
if cell_type == "code":
language = "python"
if "language_info" in notebook.get("metadata", {}):
language = notebook["metadata"]["language_info"].get(
"name",
"python",
)
lines_in_cell = source.count("\n") + 1
regions.append(
LanguageRegion(
language=language,
start_pos=current_pos,
end_pos=current_pos + len(source),
start_line=current_line,
end_line=current_line + lines_in_cell - 1,
embedding_type=EmbeddedLanguageType.SCRIPT,
parent_language="jupyter",
),
)
elif cell_type == "markdown":
lines_in_cell = source.count("\n") + 1
regions.append(
LanguageRegion(
language="markdown",
start_pos=current_pos,
end_pos=current_pos + len(source),
start_line=current_line,
end_line=current_line + lines_in_cell - 1,
embedding_type=EmbeddedLanguageType.DOCUMENTATION,
parent_language="jupyter",
),
)
current_pos += len(source)
current_line += source.count("\n") + 1
except json.JSONDecodeError:
regions.append(
LanguageRegion(
language="text",
start_pos=0,
end_pos=len(content),
start_line=1,
end_line=content.count("\n") + 1,
embedding_type=None,
parent_language=None,
),
)
return regions
@classmethod
def _identify_embedded_regions(
cls,
content: str,
primary_language: str,
) -> list[LanguageRegion]:
"""Identify embedded language regions in regular source files."""
regions = []
sql_pattern = "[\"\\'](\\s*SELECT\\s+.*?\\s+FROM\\s+.*?)[\"\\']"
for match in re.finditer(sql_pattern, content, re.IGNORECASE | re.DOTALL):
start_line = content[: match.start(1)].count("\n") + 1
end_line = content[: match.end(1)].count("\n") + 1
regions.append(
LanguageRegion(
language="sql",
start_pos=match.start(1),
end_pos=match.end(1),
start_line=start_line,
end_line=end_line,
embedding_type=EmbeddedLanguageType.QUERY,
parent_language=primary_language,
),
)
graphql_pattern = "gql`([^`]+)`|graphql\\([\"\\']([^\"\\']+)[\"\\']\\)"
for match in re.finditer(graphql_pattern, content):
group = match.group(1) or match.group(2)
if group:
start_line = content[: match.start()].count("\n") + 1
end_line = content[: match.end()].count("\n") + 1
regions.append(
LanguageRegion(
language="graphql",
start_pos=match.start(),
end_pos=match.end(),
start_line=start_line,
end_line=end_line,
embedding_type=EmbeddedLanguageType.QUERY,
parent_language=primary_language,
),
)
string_pattern = "[\"\\'](\\{.*?\\}|\\[.*?\\])[\"\\']"
# Collect potential JSON regions first
potential_json_regions = list(re.finditer(string_pattern, content, re.DOTALL))
# Process JSON validation outside the loop
for match in potential_json_regions:
try:
json.loads(match.group(1))
start_line = content[: match.start(1)].count("\n") + 1
end_line = content[: match.end(1)].count("\n") + 1
regions.append(
LanguageRegion(
language="json",
start_pos=match.start(1),
end_pos=match.end(1),
start_line=start_line,
end_line=end_line,
embedding_type=EmbeddedLanguageType.CONFIGURATION,
parent_language=primary_language,
),
)
except (json.JSONDecodeError, ValueError):
# Not valid JSON, skip
pass
return regions
def process_mixed_file( # type: ignore[override]
self,
file_path: str,
_primary_language: str,
content: str | None = None,
) -> list[CodeChunk]:
"""Process files with embedded languages."""
if content is None:
with Path(file_path).open(encoding="utf-8") as f:
content = f.read()
chunks = []
regions = self.identify_language_regions(file_path, content)
for region in regions:
if region.language not in self._supported_languages:
continue
region_content = content[region.start_pos : region.end_pos]
try:
parser = get_parser(region.language)
parser.parse(region_content.encode())
# BUG-1 fix: process_mixed_file holds in-memory region content,
# so parse it in-memory via chunk_text(text, language, file_path).
# The previous chunk_file(file_path=..., content=..., language=...)
# call did not match core.chunk_file(path, language, ...) and raised
# TypeError on every mixed file. chunk_file stays imported only as
# the patch target for tests/test_multi_language.py.
region_chunks = chunk_text(
region_content,
region.language,
file_path=file_path,
)
# region.start_pos is a CHARACTER index into ``content`` (region
# detection slices/counts on the str), but chunk byte offsets are
# UTF-8 BYTE positions. Convert the region's char prefix to its
# byte length so multibyte text before the region does not skew
# the offsets (Codex panel finding: char index added to a byte
# offset yielded a wrong slice-back on non-ASCII content).
region_byte_start = len(content[: region.start_pos].encode("utf-8"))
for chunk in region_chunks:
chunk.start_line += region.start_line - 1
chunk.end_line += region.start_line - 1
chunk.byte_start += region_byte_start
chunk.byte_end += region_byte_start
# node_id embeds byte_start, so it MUST be recomputed after
# the offset shift — otherwise two identical functions in
# separate fences collapse to the same id despite distinct
# final offsets (Codex panel finding).
chunk.node_id = compute_node_id(
file_path,
chunk.language,
chunk.qualified_route or chunk.parent_route,
chunk.byte_start,
chunk.content,
)
chunk.chunk_id = chunk.node_id
if region.embedding_type:
chunk.metadata["embedding_type"] = region.embedding_type.value
if region.parent_language:
chunk.metadata["parent_language"] = region.parent_language
chunks.append(chunk)
except (FileNotFoundError, IndexError, KeyError) as e:
chunk = CodeChunk(
language=region.language,
file_path=file_path,
node_type="region",
start_line=region.start_line,
end_line=region.end_line,
byte_start=region.start_pos,
byte_end=region.end_pos,
parent_context="",
content=region_content,
metadata={
"embedding_type": (
region.embedding_type.value
if region.embedding_type
else None
),
"parent_language": region.parent_language,
"parse_error": str(e),
},
)
chunks.append(chunk)
return chunks
@staticmethod
def extract_embedded_code(
content: str,
host_language: str,
target_language: str,
) -> list[tuple[str, int, int]]:
"""Extract embedded code snippets."""
snippets: list[tuple[str, int, int]] = []
if host_language == "html" and target_language == "javascript":
pattern = "<script[^>]*>(.*?)</script>"
snippets.extend(
(match.group(1), match.start(1), match.end(1))
for match in re.finditer(pattern, content, re.DOTALL | re.IGNORECASE)
)
event_pattern = "on\\w+\\s*=\\s*[\"\\']([^\"\\']+)[\"\\']"
snippets.extend(
(match.group(1), match.start(1), match.end(1))
for match in re.finditer(event_pattern, content)
)
elif host_language == "html" and target_language == "css":
pattern = "<style[^>]*>(.*?)</style>"
snippets.extend(
(match.group(1), match.start(1), match.end(1))
for match in re.finditer(pattern, content, re.DOTALL | re.IGNORECASE)
)
style_pattern = "style\\s*=\\s*[\"\\']([^\"\\']+)[\"\\']"
snippets.extend(
(match.group(1), match.start(1), match.end(1))
for match in re.finditer(style_pattern, content)
)
elif host_language == "markdown" and target_language:
pattern = f"```{target_language}\\n(.*?)```"
snippets.extend(
(match.group(1), match.start(1), match.end(1))
for match in re.finditer(pattern, content, re.DOTALL)
)
elif target_language == "sql":
sql_patterns = [
"[\"\\'](\\s*SELECT\\s+.*?\\s+FROM\\s+.*?)[\"\\']",
"[\"\\'](\\s*INSERT\\s+INTO\\s+.*?)[\"\\']",
"[\"\\'](\\s*UPDATE\\s+.*?\\s+SET\\s+.*?)[\"\\']",
"[\"\\'](\\s*DELETE\\s+FROM\\s+.*?)[\"\\']",
]
for pattern in sql_patterns:
snippets.extend(
(match.group(1), match.start(1), match.end(1))
for match in re.finditer(
pattern,
content,
re.IGNORECASE | re.DOTALL,
)
)
elif target_language == "graphql":
patterns = [
"gql`([^`]+)`",
"graphql\\([\"\\']([^\"\\']+)[\"\\']\\)",
"query\\s*=\\s*[\"\\']([^\"\\']+)[\"\\']",
]
for pattern in patterns:
snippets.extend(
(match.group(1), match.start(1), match.end(1))
for match in re.finditer(pattern, content)
)
return snippets
@classmethod
def cross_language_references(
cls,
chunks: list[CodeChunk],
) -> list[CrossLanguageReference]:
"""Find references across language boundaries."""
references: list[CrossLanguageReference] = []
chunks_by_name = defaultdict(list)
api_endpoints = defaultdict(list)
imports_exports = defaultdict(list)
for chunk in chunks:
if chunk.node_type in {
"function_definition",
"class_definition",
"method_definition",
}:
name_match = re.search(
r"(?:function|class|def)\s+(\w+)",
chunk.content,
)
if name_match:
name = name_match.group(1)
chunks_by_name[name].append(chunk)
if chunk.language in {
"python",
"javascript",
"typescript",
"java",
}:
endpoint_patterns = [
"[\"\\']([/\\w\\-\\{\\}:]+)[\"\\']",
"@\\w+\\([\"\\']([/\\w\\-\\{\\}:]+)[\"\\']",
]
for pattern in endpoint_patterns:
for match in re.finditer(pattern, chunk.content):
path = match.group(1)
if path.startswith("/"):
api_endpoints[path].append(chunk)
if chunk.language in {"javascript", "typescript", "python"}:
import_patterns = [
"import\\s+.*?\\s+from\\s+[\"\\']([^\"\\']+)[\"\\']",
"from\\s+([^\\s]+)\\s+import",
"require\\([\"\\']([^\"\\']+)[\"\\']\\)",
]
for pattern in import_patterns:
for match in re.finditer(pattern, chunk.content):
module = match.group(1)
imports_exports[module].append(chunk)
for chunk in chunks:
if chunk.language in {"javascript", "typescript"}:
api_call_patterns = [
"fetch\\([\"\\']([/\\w\\-\\{\\}:]+)[\"\\']",
"fetch\\(`([/\\w\\-\\{\\}:]+)",
"axios\\.\\w+\\([\"\\']([/\\w\\-\\{\\}:]+)[\"\\']",
"axios\\.\\w+\\(`([/\\w\\-\\{\\}:]+)",
"\\$\\.ajax\\(.*?url:\\s*[\"\\']([/\\w\\-\\{\\}:]+)[\"\\']",
]
for pattern in api_call_patterns:
for match in re.finditer(pattern, chunk.content):
endpoint = match.group(1)
if endpoint in api_endpoints:
references.extend(
CrossLanguageReference(
source_chunk=chunk,
target_chunk=target_chunk,
reference_type="api_call",
confidence=0.8,
)
for target_chunk in api_endpoints[endpoint]
if target_chunk.language != chunk.language
)
if chunk.node_type in {
"interface_declaration",
"type_alias_declaration",
"struct_declaration",
"class_definition",
}:
type_patterns = [
r"(?:interface|type|class|struct)\s+(\w+)",
r"type\s+(\w+)\s+struct",
]
type_name = None
for pattern in type_patterns:
type_match = re.search(pattern, chunk.content)
if type_match:
type_name = type_match.group(1)
break
if type_name:
for other_chunk in chunks:
if (
other_chunk != chunk
and other_chunk.language != chunk.language
) and other_chunk.node_type in {
"interface_declaration",
"type_alias_declaration",
"struct_declaration",
"class_definition",
}:
for pattern in type_patterns:
other_match = re.search(pattern, other_chunk.content)
if (
other_match
and other_match.group(
1,
)
== type_name
):
references.append(
CrossLanguageReference(
source_chunk=chunk,
target_chunk=other_chunk,
reference_type="shared_type",
confidence=0.6,
),
)
break
if "sql" in chunk.content.lower() or "query" in chunk.content.lower():
table_patterns = [
r"FROM\s+(\w+)",
r"INSERT\s+INTO\s+(\w+)",
r"UPDATE\s+(\w+)",
r"CREATE\s+TABLE\s+(\w+)",
]
for pattern in table_patterns:
for match in re.finditer(pattern, chunk.content, re.IGNORECASE):
table_name = match.group(1)
references.extend(
CrossLanguageReference(
source_chunk=chunk,
target_chunk=other_chunk,
reference_type="database_reference",
confidence=0.5,
)
for other_chunk in chunks
if (
other_chunk != chunk
and table_name in other_chunk.content
)
and other_chunk.language != chunk.language
)
return references
def group_by_feature(self, chunks: list[CodeChunk]) -> dict[str, list[CodeChunk]]: # type: ignore[override]
"""Group chunks from different languages by feature."""
feature_groups = defaultdict(list)
path_features = {}
for chunk in chunks:
path = Path(chunk.file_path)
parts = path.parts
feature_name = None
for i, part in enumerate(parts):
if part in {
"features",
"modules",
"components",
"domains",
"services",
} and i + 1 < len(parts):
feature_name = parts[i + 1]
break
if feature_name:
path_features[chunk.chunk_id] = feature_name
feature_groups[feature_name].append(chunk)
name_patterns = defaultdict(list)
for chunk in chunks:
if chunk.node_type in {"class_definition", "function_definition"}:
name_match = re.search(
r"(?:class|function|def)\s+(\w+)",
chunk.content,
)
if name_match:
name = name_match.group(1)
base_name = re.sub(
r"(Controller|Service|Repository|Component|Model|View)$",
"",
name,
)
name_patterns[base_name.lower()].append(chunk)
for base_name, name_chunks in name_patterns.items():
if len(name_chunks) > 1:
merged = False
# Find the most appropriate feature group to merge with
best_feature = None
best_match_count = 0
for feature_name, feature_chunks in feature_groups.items():
match_count = sum(
1 for chunk in name_chunks if chunk in feature_chunks
)
if match_count > best_match_count:
best_match_count = match_count
best_feature = feature_name
# If we found a good feature group to merge with, do it
if best_feature and best_match_count > 0:
feature_chunks = feature_groups[best_feature]
for chunk in name_chunks:
if chunk not in feature_chunks:
feature_chunks.append(chunk)
merged = True
# Also check if the base_name matches any existing feature name
if not merged:
for feature_name in feature_groups:
if (
base_name.lower() in feature_name.lower()
or feature_name.lower() in base_name.lower()
):
feature_chunks = feature_groups[feature_name]
for chunk in name_chunks:
if chunk not in feature_chunks:
feature_chunks.append(chunk)
merged = True
break
if not merged:
feature_groups[f"entity_{base_name}"] = name_chunks
references = self.cross_language_references(chunks)
reference_groups = defaultdict(set)
for ref in references:
source_feature = None
target_feature = None
for feature, feature_chunks in feature_groups.items():
if ref.source_chunk in feature_chunks:
source_feature = feature
if ref.target_chunk in feature_chunks:
target_feature = feature
if source_feature and target_feature and source_feature != target_feature:
reference_groups[source_feature].add(target_feature)
reference_groups[target_feature].add(source_feature)
for feature, related in reference_groups.items():
if feature in feature_groups:
for chunk in feature_groups[feature]:
chunk.metadata["related_features"] = list(related)
return dict(feature_groups)
"""Multi-language project processing implementation."""
import json
import os
import re
from collections import defaultdict
from pathlib import Path
from typing import Any, ClassVar
from .core import (
chunk_file,
chunk_text,
) # noqa: F401 (chunk_file kept for test patch target)
from .interfaces.multi_language import (
CrossLanguageReference,
EmbeddedLanguageType,
LanguageDetector,
LanguageRegion,
MultiLanguageProcessor,
ProjectAnalyzer,
)
from .parser import get_parser, list_languages
from .types import CodeChunk, compute_node_id
try:
pass
except ImportError:
def list_languages() -> list[str]:
return ["python", "javascript", "typescript", "java", "go", "rust", "c", "cpp"]
def get_parser(_language: Any, _config: Any = None) -> Any: # type: ignore[misc]
raise ImportError("Tree-sitter parser not available")
def chunk_file(_path: Any, _language: Any = None, **_kwargs: Any) -> Any: # type: ignore[misc]
raise ImportError("Chunker not available")
def chunk_text(_text: Any, _language: Any = None, **_kwargs: Any) -> Any: # type: ignore[misc]
raise ImportError("Chunker not available")
class LanguageDetectorImpl(LanguageDetector):
"""Detect programming languages in files and content."""
EXTENSIONS: ClassVar[dict[str, str]] = {
".py": "python",
".js": "javascript",
".jsx": "javascript",
".ts": "typescript",
".tsx": "typescript",
".java": "java",
".cpp": "cpp",
".cc": "cpp",
".cxx": "cpp",
".c": "c",
".h": "c",
".hpp": "cpp",
".rs": "rust",
".go": "go",
".rb": "ruby",
".php": "php",
".swift": "swift",
".kt": "kotlin",
".scala": "scala",
".r": "r",
".m": "objc",
".mm": "objc",
".cs": "csharp",
".vb": "vb",
".fs": "fsharp",
".ml": "ocaml",
".lua": "lua",
".pl": "perl",
".sh": "bash",
".bash": "bash",
".zsh": "bash",
".fish": "bash",
".ps1": "powershell",
".psm1": "powershell",
".html": "html",
".htm": "html",
".xml": "xml",
".css": "css",
".scss": "scss",
".sass": "sass",
".less": "less",
".json": "json",
".yaml": "yaml",
".yml": "yaml",
".toml": "toml",
".ini": "ini",
".cfg": "ini",
".md": "markdown",
".markdown": "markdown",
".rst": "rst",
".tex": "latex",
".sql": "sql",
".graphql": "graphql",
".gql": "graphql",
".ipynb": "jupyter",
Xet Storage Details
- Size:
- 53.2 kB
- Xet hash:
- 98b3b9cb2ff88cf5a0534f89f0e32441ed313bc67cf902e4cc582e7aca42dabb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.