Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /multi_language_demo.py
| #!/usr/bin/env python3 | |
| """Demo script for multi-language processing capabilities.""" | |
| import tempfile | |
| from pathlib import Path | |
| from chunker.multi_language import ( | |
| LanguageDetectorImpl, | |
| MultiLanguageProcessorImpl, | |
| ProjectAnalyzerImpl, | |
| ) | |
| from chunker.types import CodeChunk | |
| def demo_language_detection(): | |
| """Demonstrate language detection capabilities.""" | |
| print("=== Language Detection Demo ===\n") | |
| detector = LanguageDetectorImpl() | |
| # Test content samples | |
| samples = { | |
| "Python": ''' | |
| def calculate_fibonacci(n): | |
| """Calculate nth Fibonacci number.""" | |
| if n <= 1: | |
| return n | |
| return calculate_fibonacci(n-1) + calculate_fibonacci(n-2) | |
| ''', | |
| "JavaScript/React": """ | |
| import React, { useState } from 'react'; | |
| const Counter = () => { | |
| const [count, setCount] = useState(0); | |
| return ( | |
| <div className="counter"> | |
| <h1>Count: {count}</h1> | |
| <button onClick={() => setCount(count + 1)}> | |
| Increment | |
| </button> | |
| </div> | |
| ); | |
| }; | |
| export default Counter; | |
| """, | |
| "HTML with embedded JS/CSS": """ | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <style> | |
| .highlight { | |
| background-color: yellow; | |
| font-weight: bold; | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <h1 id="title">Hello World</h1> | |
| <script> | |
| document.getElementById('title').onclick = function() { | |
| this.classList.toggle('highlight'); | |
| }; | |
| </script> | |
| </body> | |
| </html> | |
| """, | |
| } | |
| for name, content in samples.items(): | |
| lang, confidence = detector.detect_from_content(content) | |
| print(f"{name}: Detected as {lang} (confidence: {confidence:.2f})") | |
| # Also check for multiple languages | |
| languages = detector.detect_multiple(content) | |
| if len(languages) > 1: | |
| print( | |
| f" Multiple languages detected: {', '.join(f'{l}({p:.1%})' for l, p in languages)}", | |
| ) | |
| print() | |
| def demo_embedded_language_detection(): | |
| """Demonstrate detection of embedded languages.""" | |
| print("=== Embedded Language Detection Demo ===\n") | |
| processor = MultiLanguageProcessorImpl() | |
| # Python with embedded SQL | |
| python_sql = ''' | |
| import sqlite3 | |
| def get_active_users(conn): | |
| """Get all active users from database.""" | |
| query = """ | |
| SELECT u.id, u.username, u.email, p.name as profile_name | |
| FROM users u | |
| JOIN profiles p ON u.id = p.user_id | |
| WHERE u.is_active = 1 | |
| ORDER BY u.created_at DESC | |
| """ | |
| cursor = conn.execute(query) | |
| return cursor.fetchall() | |
| ''' | |
| regions = processor.identify_language_regions("db_query.py", python_sql) | |
| print("Python file with embedded SQL:") | |
| for region in regions: | |
| print(f" - {region.language} ({region.start_line}-{region.end_line})") | |
| if region.embedding_type: | |
| print(f" Type: {region.embedding_type.value}") | |
| # Extract just the SQL | |
| sql_snippets = processor.extract_embedded_code(python_sql, "python", "sql") | |
| print(f"\nExtracted {len(sql_snippets)} SQL snippet(s)") | |
| print() | |
| def demo_cross_language_references(): | |
| """Demonstrate cross-language reference detection.""" | |
| print("=== Cross-Language Reference Detection Demo ===\n") | |
| processor = MultiLanguageProcessorImpl() | |
| # Create sample chunks representing a simple API | |
| chunks = [ | |
| # Python backend | |
| CodeChunk( | |
| language="python", | |
| file_path="backend/api/auth.py", | |
| node_type="function_definition", | |
| start_line=10, | |
| end_line=20, | |
| byte_start=200, | |
| byte_end=400, | |
| parent_context="", | |
| content=''' | |
| @app.route('/api/auth/login', methods=['POST']) | |
| def login(): | |
| """Handle user login.""" | |
| data = request.get_json() | |
| username = data.get('username') | |
| password = data.get('password') | |
| user = User.query.filter_by(username=username).first() | |
| if user and user.check_password(password): | |
| token = generate_token(user.id) | |
| return jsonify({'token': token, 'user_id': user.id}) | |
| return jsonify({'error': 'Invalid credentials'}), 401 | |
| ''', | |
| ), | |
| # TypeScript frontend | |
| CodeChunk( | |
| language="typescript", | |
| file_path="frontend/src/services/auth.ts", | |
| node_type="function_definition", | |
| start_line=5, | |
| end_line=15, | |
| byte_start=100, | |
| byte_end=300, | |
| parent_context="", | |
| content=""" | |
| export async function login(username: string, password: string): Promise<AuthResponse> { | |
| const response = await fetch('/api/auth/login', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify({ username, password }) | |
| }); | |
| if (!response.ok) { | |
| throw new Error('Login failed'); | |
| } | |
| return response.json(); | |
| } | |
| """, | |
| ), | |
| # Shared TypeScript interface | |
| CodeChunk( | |
| language="typescript", | |
| file_path="shared/types/auth.ts", | |
| node_type="interface_declaration", | |
| start_line=1, | |
| end_line=5, | |
| byte_start=0, | |
| byte_end=100, | |
| parent_context="", | |
| content=""" | |
| export interface AuthResponse { | |
| token: string; | |
| user_id: number; | |
| } | |
| """, | |
| ), | |
| ] | |
| references = processor.cross_language_references(chunks) | |
| print(f"Found {len(references)} cross-language reference(s):") | |
| for ref in references: | |
| print(f" - {ref.source_chunk.language} -> {ref.target_chunk.language}") | |
| print(f" Type: {ref.reference_type}") | |
| print(f" Source: {Path(ref.source_chunk.file_path).name}") | |
| print(f" Target: {Path(ref.target_chunk.file_path).name}") | |
| print(f" Confidence: {ref.confidence:.2f}") | |
| print() | |
| def demo_feature_grouping(): | |
| """Demonstrate grouping chunks by feature.""" | |
| print("=== Feature Grouping Demo ===\n") | |
| processor = MultiLanguageProcessorImpl() | |
| # Create chunks representing different features | |
| chunks = [ | |
| # User feature - backend | |
| CodeChunk( | |
| language="python", | |
| file_path="features/users/backend/user_service.py", | |
| node_type="class_definition", | |
| start_line=1, | |
| end_line=50, | |
| byte_start=0, | |
| byte_end=1000, | |
| parent_context="", | |
| content="class UserService:\n def get_user(self, user_id): pass", | |
| ), | |
| CodeChunk( | |
| language="python", | |
| file_path="features/users/backend/user_model.py", | |
| node_type="class_definition", | |
| start_line=1, | |
| end_line=30, | |
| byte_start=0, | |
| byte_end=600, | |
| parent_context="", | |
| content="class User:\n pass", | |
| ), | |
| # User feature - frontend | |
| CodeChunk( | |
| language="typescript", | |
| file_path="features/users/frontend/UserProfile.tsx", | |
| node_type="function_definition", | |
| start_line=1, | |
| end_line=40, | |
| byte_start=0, | |
| byte_end=800, | |
| parent_context="", | |
| content="export const UserProfile: React.FC = () => {}", | |
| ), | |
| # Auth feature | |
| CodeChunk( | |
| language="python", | |
| file_path="features/auth/backend/auth_service.py", | |
| node_type="class_definition", | |
| start_line=1, | |
| end_line=60, | |
| byte_start=0, | |
| byte_end=1200, | |
| parent_context="", | |
| content="class AuthService:\n def login(self): pass", | |
| ), | |
| # Shared user-related component | |
| CodeChunk( | |
| language="java", | |
| file_path="shared/controllers/UserController.java", | |
| node_type="class_definition", | |
| start_line=1, | |
| end_line=80, | |
| byte_start=0, | |
| byte_end=1600, | |
| parent_context="", | |
| content="public class UserController {}", | |
| ), | |
| ] | |
| groups = processor.group_by_feature(chunks) | |
| print(f"Grouped {len(chunks)} chunks into {len(groups)} groups:\n") | |
| for group_name, group_chunks in groups.items(): | |
| print(f"{group_name}:") | |
| for chunk in group_chunks: | |
| print(f" - {Path(chunk.file_path).name} ({chunk.language})") | |
| print() | |
| def demo_project_analysis(): | |
| """Demonstrate project structure analysis.""" | |
| print("=== Project Analysis Demo ===\n") | |
| analyzer = ProjectAnalyzerImpl() | |
| # Create a mock project structure | |
| with tempfile.TemporaryDirectory() as tmpdir: | |
| # Create project structure | |
| project_root = Path(tmpdir) / "my-fullstack-app" | |
| project_root.mkdir() | |
| # Backend structure | |
| backend = project_root / "backend" | |
| backend.mkdir() | |
| (backend / "requirements.txt").write_text("flask==2.0.0\nsqlalchemy==1.4.0\n") | |
| api_dir = backend / "api" | |
| api_dir.mkdir() | |
| (api_dir / "users.py").write_text( | |
| "from flask import Blueprint\n\nusers_bp = Blueprint('users', __name__)", | |
| ) | |
| (api_dir / "auth.py").write_text( | |
| "from flask import Blueprint\n\nauth_bp = Blueprint('auth', __name__)", | |
| ) | |
| # Frontend structure | |
| frontend = project_root / "frontend" | |
| frontend.mkdir() | |
| (frontend / "package.json").write_text( | |
| '{"name": "my-app", "dependencies": {"react": "^18.0.0"}}', | |
| ) | |
| src_dir = frontend / "src" | |
| src_dir.mkdir() | |
| (src_dir / "App.tsx").write_text( | |
| "import React from 'react';\n\nexport const App = () => <div>Hello</div>;", | |
| ) | |
| (src_dir / "index.js").write_text( | |
| "import { App } from './App';\n\nReactDOM.render(<App />, document.getElementById('root'));", | |
| ) | |
| # Tests | |
| tests = project_root / "tests" | |
| tests.mkdir() | |
| (tests / "test_api.py").write_text( | |
| "import pytest\n\ndef test_users_endpoint():\n pass", | |
| ) | |
| # Analyze the project | |
| analysis = analyzer.analyze_structure(str(project_root)) | |
| print(f"Project: {analysis['project_path']}") | |
| print(f"Type: {analysis['project_type']}") | |
| print(f"Total files: {analysis['file_count']}") | |
| print(f"Total lines: {analysis['total_lines']}") | |
| print("\nLanguages detected:") | |
| for lang, count in analysis["languages"].items(): | |
| print(f" - {lang}: {count} files") | |
| print("\nProject structure:") | |
| for key, value in analysis["structure"].items(): | |
| print(f" - {key}: {value}") | |
| print("\nFramework indicators:") | |
| for indicator in analysis["framework_indicators"]: | |
| print(f" - {indicator}") | |
| if __name__ == "__main__": | |
| demo_language_detection() | |
| demo_embedded_language_detection() | |
| demo_cross_language_references() | |
| demo_feature_grouping() | |
| demo_project_analysis() | |
Xet Storage Details
- Size:
- 11 kB
- Xet hash:
- 29a7f871cd3cf20f5132aa38ee29787701bf338d79f508cd9d7f297796df0037
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.