Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Kolerr-Lab_supper-decoyable /decoyable /cache.py
| """Redis-based caching layer for DECOYABLE performance optimization.""" | |
| import hashlib | |
| import json | |
| import logging | |
| import os | |
| import time | |
| from functools import wraps | |
| from typing import Any, Dict, List, Optional | |
| try: | |
| import redis | |
| REDIS_AVAILABLE = True | |
| except ImportError: | |
| REDIS_AVAILABLE = False | |
| redis = None | |
| logger = logging.getLogger(__name__) | |
| class Cache: | |
| """Redis-based caching with fallback to in-memory cache.""" | |
| def __init__(self, redis_url: Optional[str] = None, default_ttl: int = 3600): | |
| """ | |
| Initialize cache with Redis backend. | |
| Args: | |
| redis_url: Redis connection URL (redis://host:port/db) | |
| default_ttl: Default time-to-live in seconds | |
| """ | |
| self.default_ttl = default_ttl | |
| self.redis_client = None | |
| self.memory_cache: Dict[str, Dict[str, Any]] = {} | |
| if REDIS_AVAILABLE and redis_url: | |
| try: | |
| self.redis_client = redis.from_url(redis_url) | |
| # Test connection | |
| self.redis_client.ping() | |
| logger.info("Redis cache initialized successfully") | |
| except Exception as e: | |
| logger.warning(f"Failed to connect to Redis: {e}. Using memory cache.") | |
| self.redis_client = None | |
| else: | |
| logger.info("Redis not available or not configured. Using memory cache.") | |
| def _make_key(self, *args, **kwargs) -> str: | |
| """Create a consistent cache key from arguments.""" | |
| # Sort kwargs for consistent key generation | |
| sorted_kwargs = sorted(kwargs.items()) | |
| key_data = json.dumps({"args": args, "kwargs": sorted_kwargs}, sort_keys=True) | |
| return hashlib.sha256(key_data.encode()).hexdigest() | |
| def get(self, key: str) -> Optional[Any]: | |
| """Get value from cache.""" | |
| if self.redis_client: | |
| try: | |
| data = self.redis_client.get(key) | |
| if data: | |
| # Safe: JSON deserialization from trusted Redis cache with type validation | |
| try: | |
| parsed_data = json.loads(data.decode("utf-8")) | |
| # Basic validation - ensure it's a dict or list | |
| if not isinstance(parsed_data, (dict, list)): | |
| logger.warning(f"Invalid cache data type for key {key}: {type(parsed_data)}") | |
| return None | |
| return parsed_data | |
| except (json.JSONDecodeError, TypeError) as e: | |
| logger.warning(f"Invalid JSON in cache for key {key}: {e}") | |
| return None | |
| except Exception as e: | |
| logger.warning(f"Redis get error: {e}") | |
| return None | |
| else: | |
| # Memory cache fallback | |
| cache_entry = self.memory_cache.get(key) | |
| if cache_entry: | |
| if time.time() < cache_entry["expires"]: | |
| return cache_entry["value"] | |
| else: | |
| # Expired, remove it | |
| del self.memory_cache[key] | |
| return None | |
| def set(self, key: str, value: Any, ttl: Optional[int] = None) -> bool: | |
| """Set value in cache with optional TTL.""" | |
| ttl = ttl or self.default_ttl | |
| if self.redis_client: | |
| try: | |
| data = json.dumps(value) | |
| return bool(self.redis_client.setex(key, ttl, data)) | |
| except Exception as e: | |
| logger.warning(f"Redis set error: {e}") | |
| return False | |
| else: | |
| # Memory cache fallback | |
| self.memory_cache[key] = {"value": value, "expires": time.time() + ttl} | |
| return True | |
| def delete(self, key: str) -> bool: | |
| """Delete key from cache.""" | |
| if self.redis_client: | |
| try: | |
| return bool(self.redis_client.delete(key)) | |
| except Exception as e: | |
| logger.warning(f"Redis delete error: {e}") | |
| return False | |
| else: | |
| return bool(self.memory_cache.pop(key, None)) | |
| def clear(self) -> bool: | |
| """Clear all cache entries.""" | |
| if self.redis_client: | |
| try: | |
| return bool(self.redis_client.flushdb()) | |
| except Exception as e: | |
| logger.warning(f"Redis clear error: {e}") | |
| return False | |
| else: | |
| self.memory_cache.clear() | |
| return True | |
| def exists(self, key: str) -> bool: | |
| """Check if key exists in cache.""" | |
| if self.redis_client: | |
| try: | |
| return bool(self.redis_client.exists(key)) | |
| except Exception as e: | |
| logger.warning(f"Redis exists error: {e}") | |
| return False | |
| else: | |
| cache_entry = self.memory_cache.get(key) | |
| if cache_entry: | |
| if time.time() < cache_entry["expires"]: | |
| return True | |
| else: | |
| # Expired, remove it | |
| del self.memory_cache[key] | |
| return False | |
| # Global cache instance | |
| _cache_instance: Optional[Cache] = None | |
| def get_cache() -> Cache: | |
| """Get or create global cache instance.""" | |
| global _cache_instance | |
| if _cache_instance is None: | |
| redis_url = os.getenv("REDIS_URL") | |
| _cache_instance = Cache(redis_url=redis_url) | |
| return _cache_instance | |
| def cached(ttl: Optional[int] = None, key_prefix: str = ""): | |
| """ | |
| Decorator to cache function results. | |
| Args: | |
| ttl: Time-to-live in seconds (uses cache default if None) | |
| key_prefix: Prefix for cache keys to avoid collisions | |
| """ | |
| def decorator(func): | |
| def wrapper(*args, **kwargs): | |
| cache = get_cache() | |
| cache_key = f"{key_prefix}:{func.__name__}:{cache._make_key(*args, **kwargs)}" | |
| # Try to get from cache first | |
| cached_result = cache.get(cache_key) | |
| if cached_result is not None: | |
| logger.debug(f"Cache hit for {func.__name__}") | |
| return cached_result | |
| # Cache miss, execute function | |
| logger.debug(f"Cache miss for {func.__name__}") | |
| result = func(*args, **kwargs) | |
| # Cache the result | |
| cache.set(cache_key, result, ttl) | |
| return result | |
| return wrapper | |
| return decorator | |
| # Cache-enabled scanner functions | |
| # Cache for 30 minutes | |
| def scan_secrets_cached(paths: List[str]) -> List[Dict[str, Any]]: | |
| """Cached version of secrets scanning.""" | |
| from decoyable.scanners import secrets | |
| findings = secrets.scan_paths(paths) | |
| results = [] | |
| for finding in findings: | |
| results.append( | |
| { | |
| "filename": finding.filename, | |
| "lineno": finding.lineno, | |
| "secret_type": finding.secret_type, | |
| "masked": finding.masked(), | |
| "context": finding.context, | |
| } | |
| ) | |
| return results | |
| # Cache for 1 hour | |
| def scan_dependencies_cached(path: str) -> Dict[str, Any]: | |
| """Cached version of dependency scanning.""" | |
| from decoyable.scanners import deps | |
| missing_imports, import_mapping = deps.missing_dependencies(path) | |
| results = [] | |
| for imp in sorted(missing_imports): | |
| providers = import_mapping.get(imp, []) | |
| results.append({"import": imp, "providers": providers}) | |
| return {"missing_dependencies": results, "count": len(results)} | |
| # Cache for 30 minutes | |
| def scan_sast_cached(path: str) -> Dict[str, Any]: | |
| """Cached version of SAST scanning.""" | |
| from decoyable.scanners import sast | |
| return sast.scan_sast(path) | |
| def get_cache_stats() -> Dict[str, Any]: | |
| """Get cache statistics and information.""" | |
| cache = get_cache() | |
| stats = { | |
| "cache_type": "redis" if cache.redis_client else "memory", | |
| "redis_available": REDIS_AVAILABLE, | |
| } | |
| if cache.redis_client: | |
| try: | |
| info = cache.redis_client.info() | |
| stats.update( | |
| { | |
| "redis_connected": True, | |
| "redis_memory_used": info.get("used_memory_human", "unknown"), | |
| "redis_keys_count": cache.redis_client.dbsize(), | |
| "redis_uptime_days": info.get("uptime_in_days", 0), | |
| } | |
| ) | |
| except Exception as e: | |
| stats.update({"redis_connected": False, "redis_error": str(e)}) | |
| else: | |
| stats.update( | |
| { | |
| "memory_cache_entries": len(cache.memory_cache), | |
| } | |
| ) | |
| return stats | |
| def warmup_cache(target_path: str = ".") -> Dict[str, Any]: | |
| """ | |
| Warm up the cache by pre-computing common scan results. | |
| Returns statistics about cache warmup. | |
| """ | |
| logger.info("Starting cache warmup...") | |
| start_time = time.time() | |
| operations = [] | |
| try: | |
| # Warm up secrets scan | |
| secrets_result = scan_secrets_cached([target_path]) | |
| operations.append({"operation": "secrets_scan", "cached": True, "items": len(secrets_result)}) | |
| # Warm up dependencies scan | |
| deps_result = scan_dependencies_cached(target_path) | |
| operations.append({"operation": "deps_scan", "cached": True, "items": deps_result["count"]}) | |
| # Warm up SAST scan | |
| sast_result = scan_sast_cached(target_path) | |
| operations.append( | |
| { | |
| "operation": "sast_scan", | |
| "cached": True, | |
| "vulnerabilities": sast_result["summary"]["total_vulnerabilities"], | |
| } | |
| ) | |
| except Exception as e: | |
| logger.error(f"Cache warmup failed: {e}") | |
| return {"status": "error", "error": str(e)} | |
| duration = time.time() - start_time | |
| return { | |
| "status": "success", | |
| "duration_seconds": round(duration, 2), | |
| "operations": operations, | |
| "cache_stats": get_cache_stats(), | |
| } | |
Xet Storage Details
- Size:
- 10.1 kB
- Xet hash:
- d9560ec9582148805f6d00881e74a4dcb26ff1e7da1073b0de8162d590e35b6c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.