Buckets:

download
raw
17.1 kB
#!/usr/bin/env python3
"""
Comprehensive workspace analysis and diagnostics
Establishes health baselines and best practices across all repos
"""
import json
import re
import sys
from datetime import datetime
from pathlib import Path
from typing import Any
# Configuration
REPOS = {
"Apps": {
"path": "e:\\projects\\Apps",
"type": "hybrid",
"primary_langs": ["python", "typescript"],
},
"grid": {"path": "e:\\grid", "type": "python", "primary_langs": ["python"]},
"EUFLE": {"path": "e:\\projects\\EUFLE", "type": "python", "primary_langs": ["python"]},
"pipeline": {"path": "e:\\projects\\pipeline", "type": "python", "primary_langs": ["python"]},
}
ANALYSIS_OUTPUTS = Path("e:\\analysis_outputs")
ANALYSIS_OUTPUTS.mkdir(exist_ok=True)
# ==================== Phase 1: Type Safety ====================
class TypeSafetyAnalyzer:
"""Analyze type safety and Pydantic v2 compliance"""
def __init__(self):
self.errors = []
self.warnings = []
self.fixed_count = 0
def check_pydantic_v2_migration(self, repo_path: Path) -> dict[str, Any]:
"""Check for Pydantic v2 compatibility issues"""
findings = {
"field_mismatches": [],
"import_issues": [],
"deprecated_patterns": [],
}
for py_file in repo_path.rglob("*.py"):
try:
with open(py_file, encoding="utf-8", errors="ignore") as f:
content = f.read()
# Check for deprecated Field() patterns
if re.search(r"Field\([^)]*default_factory[^)]*\)", content):
lines = content.split("\n")
for i, line in enumerate(lines, 1):
if "Field(" in line and "default_factory" in line:
# V2 requires lowercase
if "default_factory" in line:
findings["field_mismatches"].append(
{
"file": str(py_file.relative_to(repo_path)),
"line": i,
"issue": "Verify Field syntax for Pydantic v2",
}
)
# Check for old imports
if "from pydantic import validator" in content:
findings["deprecated_patterns"].append(
{
"file": str(py_file.relative_to(repo_path)),
"issue": "validator decorator deprecated; use field_validator",
}
)
except Exception:
pass
return findings
def check_optional_member_access(self, repo_path: Path) -> dict[str, Any]:
"""Identify unguarded optional member access"""
findings = {"unguarded_access": [], "safe_patterns": []}
for py_file in repo_path.rglob("*.py"):
try:
with open(py_file, encoding="utf-8", errors="ignore") as f:
content = f.read()
lines = content.split("\n")
# Look for patterns like obj.attr without null check
for i, line in enumerate(lines, 1):
# Skip comments and strings
if line.strip().startswith("#"):
continue
# Pattern: something.get() or ?.operator
if ".get(" in line or " if " in line and " is not None" in line:
findings["safe_patterns"].append(
{
"file": str(py_file.relative_to(repo_path)),
"line": i,
"pattern": "safe",
}
)
except Exception:
pass
return findings
def analyze_repo(self, repo_name: str, repo_path: Path) -> dict[str, Any]:
"""Analyze type safety for a single repo"""
report = {
"repo": repo_name,
"timestamp": datetime.utcnow().isoformat(),
"pydantic_migration": self.check_pydantic_v2_migration(repo_path),
"optional_access": self.check_optional_member_access(repo_path),
"summary": {},
}
# Count Python files
py_files = list(repo_path.rglob("*.py"))
report["summary"]["total_py_files"] = len(py_files)
report["summary"]["pydantic_issues"] = len(report["pydantic_migration"]["field_mismatches"])
report["summary"]["deprecated_patterns"] = len(report["pydantic_migration"]["deprecated_patterns"])
return report
# ==================== Phase 2: AI Safety & Privacy ====================
class AISafetyAuditor:
"""Audit AI safety and privacy practices"""
MODEL_PATTERNS = {
"openai": ["openai.", "gpt-", "from openai"],
"anthropic": ["anthropic.", "claude", "from anthropic"],
"ollama": ["Ollama", "ollama.", "from ollama"],
"huggingface": ["transformers", "huggingface", "HfApi"],
"local": ["local.*model", "on.*premise"],
}
PII_PATTERNS = {
"email": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
"ssn": r"\b\d{3}-\d{2}-\d{4}\b",
"credit_card": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b",
"phone": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b",
}
def detect_model_usage(self, repo_path: Path) -> dict[str, list[str]]:
"""Detect AI model usage"""
detected = {}
for py_file in repo_path.rglob("*.py"):
try:
with open(py_file, encoding="utf-8", errors="ignore") as f:
content = f.read()
for model, patterns in self.MODEL_PATTERNS.items():
for pattern in patterns:
if re.search(pattern, content, re.IGNORECASE):
if model not in detected:
detected[model] = []
detected[model].append(str(py_file.relative_to(repo_path)))
break
except Exception:
pass
return detected
def check_pii_handling(self, repo_path: Path) -> dict[str, Any]:
"""Check for PII in code and patterns"""
findings = {
"redaction_patterns": [],
"encryption_patterns": [],
"potential_pii": [],
}
for py_file in repo_path.rglob("*.py"):
try:
with open(py_file, encoding="utf-8", errors="ignore") as f:
content = f.read()
# Look for redaction/encryption patterns
if any(x in content for x in ["redact", "encrypt", "hash", "anonymize", "gdpr", "pii"]):
findings["redaction_patterns"].append(str(py_file.relative_to(repo_path)))
# Look for encryption imports
if "cryptography" in content or "Fernet" in content:
findings["encryption_patterns"].append(str(py_file.relative_to(repo_path)))
except Exception:
pass
return findings
def analyze_repo(self, repo_name: str, repo_path: Path) -> dict[str, Any]:
"""Analyze AI safety for a repo"""
return {
"repo": repo_name,
"timestamp": datetime.utcnow().isoformat(),
"model_usage": self.detect_model_usage(repo_path),
"pii_handling": self.check_pii_handling(repo_path),
"risk_assessment": {
"models_detected": len(self.detect_model_usage(repo_path)),
"has_encryption": len(self.check_pii_handling(repo_path).get("encryption_patterns", [])) > 0,
"has_redaction": len(self.check_pii_handling(repo_path).get("redaction_patterns", [])) > 0,
},
}
# ==================== Phase 3: Architecture Testing ====================
class ArchitectureTester:
"""Test architectural components and patterns"""
def test_async_patterns(self, repo_path: Path) -> dict[str, Any]:
"""Verify async/await patterns"""
findings = {"async_functions": 0, "async_issues": [], "sync_blocking_calls": []}
for py_file in repo_path.rglob("*.py"):
try:
with open(py_file, encoding="utf-8", errors="ignore") as f:
content = f.read()
# Count async functions
findings["async_functions"] += len(re.findall(r"async def ", content))
# Check for blocking calls in async context
if "async def" in content:
lines = content.split("\n")
for i, line in enumerate(lines, 1):
if "time.sleep(" in line or "open(" in line or "requests." in line:
findings["sync_blocking_calls"].append(
{
"file": str(py_file.relative_to(repo_path)),
"line": i,
"issue": "Potential blocking call in async context",
}
)
except Exception:
pass
return findings
def test_service_layer(self, repo_path: Path) -> dict[str, Any]:
"""Test service layer patterns"""
findings = {"services": [], "routers": [], "import_issues": []}
services_dir = repo_path / "backend" / "services" if (repo_path / "backend" / "services").exists() else None
if services_dir:
for service_file in services_dir.glob("*.py"):
findings["services"].append(service_file.name)
routers_dir = repo_path / "backend" / "routers" if (repo_path / "backend" / "routers").exists() else None
if routers_dir:
for router_file in routers_dir.glob("*.py"):
findings["routers"].append(router_file.name)
return findings
def analyze_repo(self, repo_name: str, repo_path: Path) -> dict[str, Any]:
"""Analyze architecture for a repo"""
return {
"repo": repo_name,
"timestamp": datetime.utcnow().isoformat(),
"async_patterns": self.test_async_patterns(repo_path),
"service_layer": self.test_service_layer(repo_path),
}
# ==================== Phase 4: Performance Baseline ====================
class PerformanceBaseliner:
"""Establish performance baselines"""
def check_codebase_metrics(self, repo_path: Path) -> dict[str, Any]:
"""Collect codebase metrics"""
metrics = {
"total_lines": 0,
"python_files": 0,
"typescript_files": 0,
"test_files": 0,
"configuration_files": 0,
"average_file_size": 0,
"largest_files": [],
}
py_files = list(repo_path.rglob("*.py"))
ts_files = list(repo_path.rglob("*.ts")) + list(repo_path.rglob("*.tsx"))
test_files = list(repo_path.rglob("*test*.py")) + list(repo_path.rglob("*.test.ts"))
metrics["python_files"] = len(py_files)
metrics["typescript_files"] = len(ts_files)
metrics["test_files"] = len(test_files)
all_lines = 0
file_sizes = []
for py_file in py_files:
try:
with open(py_file, encoding="utf-8", errors="ignore") as f:
lines = len(f.readlines())
all_lines += lines
file_sizes.append((py_file.name, lines))
except Exception:
pass
metrics["total_lines"] = all_lines
metrics["average_file_size"] = all_lines // len(py_files) if py_files else 0
metrics["largest_files"] = sorted(file_sizes, key=lambda x: x[1], reverse=True)[:5]
return metrics
def analyze_repo(self, repo_name: str, repo_path: Path) -> dict[str, Any]:
"""Analyze performance metrics for a repo"""
return {
"repo": repo_name,
"timestamp": datetime.utcnow().isoformat(),
"codebase_metrics": self.check_codebase_metrics(repo_path),
}
# ==================== Main Execution ====================
def main():
print("=" * 80)
print("COMPREHENSIVE WORKSPACE ANALYSIS & DIAGNOSTICS")
print("=" * 80)
print(f"Started: {datetime.utcnow().isoformat()}\n")
# Initialize analyzers
type_analyzer = TypeSafetyAnalyzer()
safety_auditor = AISafetyAuditor()
arch_tester = ArchitectureTester()
perf_baseline = PerformanceBaseliner()
# Storage for all results
comprehensive_report = {
"timestamp": datetime.utcnow().isoformat(),
"phase1_type_safety": {},
"phase2_ai_safety": {},
"phase3_architecture": {},
"phase4_performance": {},
"executive_summary": {},
}
# Analyze each repo
for repo_name, repo_info in REPOS.items():
repo_path = Path(repo_info["path"])
if not repo_path.exists():
print(f"[!] Repo not found: {repo_name} at {repo_path}")
continue
print(f"\n[Analyzing] {repo_name}...")
# Phase 1: Type Safety
print(" -> Phase 1: Type Safety")
type_report = type_analyzer.analyze_repo(repo_name, repo_path)
comprehensive_report["phase1_type_safety"][repo_name] = type_report
print(f" [OK] Found {type_report['summary']['total_py_files']} Python files")
# Phase 2: AI Safety
print(" -> Phase 2: AI Safety & Privacy")
safety_report = safety_auditor.analyze_repo(repo_name, repo_path)
comprehensive_report["phase2_ai_safety"][repo_name] = safety_report
models = list(safety_report["model_usage"].keys())
print(f" [OK] Models detected: {models if models else 'None'}")
# Phase 3: Architecture
print(" -> Phase 3: Architecture")
arch_report = arch_tester.analyze_repo(repo_name, repo_path)
comprehensive_report["phase3_architecture"][repo_name] = arch_report
print(f" [OK] {arch_report['async_patterns']['async_functions']} async functions")
# Phase 4: Performance
print(" -> Phase 4: Performance Baseline")
perf_report = perf_baseline.analyze_repo(repo_name, repo_path)
comprehensive_report["phase4_performance"][repo_name] = perf_report
metrics = perf_report["codebase_metrics"]
print(f" [OK] {metrics['total_lines']} lines of code across {metrics['python_files']} files")
# Generate Executive Summary
print("\n" + "=" * 80)
print("GENERATING EXECUTIVE SUMMARY")
print("=" * 80)
total_py_files = sum(
comprehensive_report["phase1_type_safety"][r]["summary"]["total_py_files"]
for r in comprehensive_report["phase1_type_safety"]
if r
)
total_issues = sum(
comprehensive_report["phase1_type_safety"][r]["summary"]["pydantic_issues"]
for r in comprehensive_report["phase1_type_safety"]
if r
)
comprehensive_report["executive_summary"] = {
"total_repos": len(REPOS),
"total_python_files": total_py_files,
"type_safety_issues": total_issues,
"models_detected": list(
set(
model
for repo_data in comprehensive_report["phase2_ai_safety"].values()
for model in repo_data.get("model_usage", {}).keys()
)
),
"async_function_count": sum(
comprehensive_report["phase3_architecture"][r]["async_patterns"]["async_functions"]
for r in comprehensive_report["phase3_architecture"]
if r
),
"total_lines_of_code": sum(
comprehensive_report["phase4_performance"][r]["codebase_metrics"]["total_lines"]
for r in comprehensive_report["phase4_performance"]
if r
),
"analysis_date": datetime.utcnow().isoformat(),
}
# Save comprehensive report
report_path = ANALYSIS_OUTPUTS / "comprehensive_analysis_report.json"
with open(report_path, "w", encoding="utf-8") as f:
json.dump(comprehensive_report, f, indent=2, default=str)
print("\n[OK] Comprehensive analysis complete!")
print(f"Report saved to: {report_path}")
print("\nKey Metrics:")
print(f" - Total Python Files: {total_py_files}")
print(f" - Type Safety Issues: {total_issues}")
print(f" - Models Detected: {comprehensive_report['executive_summary']['models_detected']}")
print(f" - Async Functions: {comprehensive_report['executive_summary']['async_function_count']}")
print(f" - Total Lines of Code: {comprehensive_report['executive_summary']['total_lines_of_code']}")
return 0
if __name__ == "__main__":
sys.exit(main())

Xet Storage Details

Size:
17.1 kB
·
Xet hash:
0d101978115113ddefc66bad2b3b5524c48b857d31e1cae8d1bae9af3e48ef1c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.