monitoring_system.py

"""
Monitoring System - Layer 4: System monitoring and analytics (Free Version)
Lightweight monitoring for Hugging Face Spaces
"""

import time
from typing import Dict, List, Any, Optional
from datetime import datetime, timedelta
from dataclasses import dataclass
import json
import pandas as pd
import numpy as np


@dataclass
class Alert:
    """Monitoring alert"""
    id: str
    severity: str  # info, warning, error
    source: str
    message: str
    timestamp: datetime
    acknowledged: bool = False
    resolved: bool = False


@dataclass
class PerformanceSnapshot:
    """Performance snapshot"""
    timestamp: datetime
    metrics: Dict[str, Any]
    agent_status: Dict[str, str]


class MonitoringSystem:
    """Lightweight monitoring system for free Hugging Face Spaces"""
    
    def __init__(self, max_history: int = 1000):
        self.alerts: List[Alert] = []
        self.performance_history: List[PerformanceSnapshot] = []
        self.max_history = max_history
        
        # System metrics
        self.system_metrics = {
            'start_time': datetime.now(),
            'total_queries': 0,
            'successful_queries': 0,
            'failed_queries': 0,
            'active_sessions': 0,
            'peak_memory_mb': 0,
            'average_response_time': 0.0
        }
        
        # Agent metrics
        self.agent_metrics = {
            'diagnostician': {'calls': 0, 'avg_time': 0.0, 'errors': 0},
            'socratic_tutor': {'calls': 0, 'avg_time': 0.0, 'errors': 0},
            'calculator': {'calls': 0, 'avg_time': 0.0, 'errors': 0},
            'knowledge_expert': {'calls': 0, 'avg_time': 0.0, 'errors': 0},
            'practice_generator': {'calls': 0, 'avg_time': 0.0, 'errors': 0},
            'assessment': {'calls': 0, 'avg_time': 0.0, 'errors': 0}
        }
        
        # Alert rules
        self.alert_rules = [
            {
                'condition': lambda metrics: metrics.get('failed_queries', 0) > 10,
                'severity': 'warning',
                'message': 'High number of failed queries detected'
            },
            {
                'condition': lambda metrics: metrics.get('average_response_time', 0) > 5.0,
                'severity': 'warning',
                'message': 'Slow response times detected'
            }
        ]
    
    def update_metrics(self):
        """Update system metrics (called periodically)"""
        # Create snapshot
        snapshot = PerformanceSnapshot(
            timestamp=datetime.now(),
            metrics=self.system_metrics.copy(),
            agent_status={agent: 'active' for agent in self.agent_metrics.keys()}
        )
        
        self.performance_history.append(snapshot)
        
        # Keep history within limits
        if len(self.performance_history) > self.max_history:
            self.performance_history = self.performance_history[-self.max_history:]
        
        # Check alerts
        self._check_alerts()
        
        # Cleanup old alerts
        self._cleanup_alerts()
    
    def record_query(self, success: bool, response_time: float = 0.0):
        """Record a query"""
        self.system_metrics['total_queries'] += 1
        
        if success:
            self.system_metrics['successful_queries'] += 1
        else:
            self.system_metrics['failed_queries'] += 1
        
        # Update average response time
        n = self.system_metrics['total_queries']
        current_avg = self.system_metrics['average_response_time']
        self.system_metrics['average_response_time'] = (
            (current_avg * (n - 1) + response_time) / n if n > 1 else response_time
        )
    
    def record_agent_call(self, agent_name: str, success: bool, execution_time: float):
        """Record an agent call"""
        if agent_name in self.agent_metrics:
            metrics = self.agent_metrics[agent_name]
            metrics['calls'] += 1
            
            # Update average time
            n = metrics['calls']
            current_avg = metrics['avg_time']
            metrics['avg_time'] = (
                (current_avg * (n - 1) + execution_time) / n if n > 1 else execution_time
            )
            
            if not success:
                metrics['errors'] += 1
    
    def get_system_metrics(self) -> Dict[str, Any]:
        """Get current system metrics"""
        # Calculate uptime
        uptime = datetime.now() - self.system_metrics['start_time']
        uptime_hours = uptime.total_seconds() / 3600
        
        # Calculate success rate
        total = self.system_metrics['total_queries']
        successful = self.system_metrics['successful_queries']
        success_rate = successful / total if total > 0 else 1.0
        
        # Calculate active agents
        active_agents = sum(1 for metrics in self.agent_metrics.values() if metrics['calls'] > 0)
        
        # Get recent performance
        recent_snapshots = [s for s in self.performance_history 
                          if s.timestamp > datetime.now() - timedelta(hours=1)]
        
        avg_response_time = self.system_metrics['average_response_time']
        
        if recent_snapshots:
            recent_times = [s.metrics.get('average_response_time', 0) for s in recent_snapshots]
            avg_response_time = sum(recent_times) / len(recent_times)
        
        return {
            'uptime_hours': uptime_hours,
            'total_queries': total,
            'successful_queries': successful,
            'failed_queries': self.system_metrics['failed_queries'],
            'success_rate': success_rate,
            'avg_response_time': avg_response_time,
            'active_agents': active_agents,
            'knowledge_coverage': self._estimate_knowledge_coverage(),
            'system_health': self._calculate_system_health()
        }
    
    def _estimate_knowledge_coverage(self) -> float:
        """Estimate knowledge coverage percentage"""
        # Simulated based on agent usage
        total_calls = sum(metrics['calls'] for metrics in self.agent_metrics.values())
        if total_calls == 0:
            return 0.7  # Default coverage
        
        # More calls to knowledge expert indicates better coverage
        knowledge_calls = self.agent_metrics.get('knowledge_expert', {}).get('calls', 0)
        coverage = 0.7 + (knowledge_calls / max(total_calls, 1)) * 0.3
        
        return min(coverage, 0.95)
    
    def _calculate_system_health(self) -> float:
        """Calculate overall system health score (0-100)"""
        # Based on success rate and response time
        success_rate = self.get_system_metrics()['success_rate']
        avg_response_time = self.get_system_metrics()['avg_response_time']
        
        # Success component (0-70 points)
        success_score = success_rate * 70
        
        # Response time component (0-30 points, faster = better)
        if avg_response_time < 1.0:
            response_score = 30
        elif avg_response_time < 3.0:
            response_score = 20
        elif avg_response_time < 5.0:
            response_score = 10
        else:
            response_score = 0
        
        health_score = success_score + response_score
        return min(health_score, 100)
    
    def _check_alerts(self):
        """Check for alert conditions"""
        metrics = self.get_system_metrics()
        
        for rule in self.alert_rules:
            if rule['condition'](metrics):
                self._create_alert(
                    severity=rule['severity'],
                    source='system',
                    message=rule['message']
                )
    
    def _create_alert(self, severity: str, source: str, message: str):
        """Create a new alert"""
        alert_id = f"alert_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{len(self.alerts)}"
        
        alert = Alert(
            id=alert_id,
            severity=severity,
            source=source,
            message=message,
            timestamp=datetime.now()
        )
        
        self.alerts.append(alert)
        print(f"🚨 Alert: {severity.upper()} - {message}")
    
    def _cleanup_alerts(self):
        """Cleanup old alerts"""
        # Keep only last 50 alerts
        if len(self.alerts) > 50:
            self.alerts = self.alerts[-50:]
        
        # Auto-resolve old info alerts (24 hours)
        cutoff = datetime.now() - timedelta(hours=24)
        for alert in self.alerts:
            if alert.severity == 'info' and alert.timestamp < cutoff:
                alert.resolved = True
    
    def get_agent_logs(self, limit: int = 20) -> pd.DataFrame:
        """Get agent activity logs"""
        logs = []
        
        for agent, metrics in self.agent_metrics.items():
            if metrics['calls'] > 0:
                logs.append({
                    'agent': agent,
                    'calls': metrics['calls'],
                    'avg_time': metrics['avg_time'],
                    'error_rate': metrics['errors'] / metrics['calls'] if metrics['calls'] > 0 else 0,
                    'status': 'active' if metrics['calls'] > 0 else 'inactive'
                })
        
        if logs:
            df = pd.DataFrame(logs)
            return df.sort_values('calls', ascending=False).head(limit)
        
        return pd.DataFrame(columns=['agent', 'calls', 'avg_time', 'error_rate', 'status'])
    
    def get_active_alerts(self) -> List[Alert]:
        """Get active (unresolved) alerts"""
        return [a for a in self.alerts if not a.resolved]
    
    def acknowledge_alert(self, alert_id: str) -> bool:
        """Acknowledge an alert"""
        for alert in self.alerts:
            if alert.id == alert_id:
                alert.acknowledged = True
                return True
        return False
    
    def resolve_alert(self, alert_id: str) -> bool:
        """Resolve an alert"""
        for alert in self.alerts:
            if alert.id == alert_id:
                alert.resolved = True
                return True
        return False
    
    def get_performance_report(self, period: str = '1h') -> Dict[str, Any]:
        """Get performance report for a period"""
        # Calculate time cutoff
        if period == '1h':
            cutoff = datetime.now() - timedelta(hours=1)
        elif period == '24h':
            cutoff = datetime.now() - timedelta(hours=24)
        else:
            cutoff = datetime.now() - timedelta(minutes=30)
        
        # Filter snapshots
        period_snapshots = [s for s in self.performance_history if s.timestamp > cutoff]
        
        if not period_snapshots:
            current_metrics = self.get_system_metrics()
            return {
                'period': period,
                'data_points': 0,
                'current_metrics': current_metrics,
                'trend': 'insufficient_data'
            }
        
        # Calculate period metrics
        total_queries = sum(s.metrics.get('total_queries', 0) for s in period_snapshots)
        successful_queries = sum(s.metrics.get('successful_queries', 0) for s in period_snapshots)
        
        response_times = [s.metrics.get('average_response_time', 0) for s in period_snapshots]
        avg_response_time = sum(response_times) / len(response_times) if response_times else 0
        
        # Calculate trend
        if len(period_snapshots) >= 2:
            first_half = period_snapshots[:len(period_snapshots)//2]
            second_half = period_snapshots[len(period_snapshots)//2:]
            
            first_rate = sum(s.metrics.get('successful_queries', 0) for s in first_half) / max(sum(s.metrics.get('total_queries', 0) for s in first_half), 1)
            second_rate = sum(s.metrics.get('successful_queries', 0) for s in second_half) / max(sum(s.metrics.get('total_queries', 0) for s in second_half), 1)
            
            if second_rate > first_rate * 1.1:
                trend = 'improving'
            elif second_rate < first_rate * 0.9:
                trend = 'declining'
            else:
                trend = 'stable'
        else:
            trend = 'stable'
        
        current_metrics = self.get_system_metrics()
        
        return {
            'period': period,
            'data_points': len(period_snapshots),
            'queries_in_period': total_queries,
            'success_rate_period': successful_queries / total_queries if total_queries > 0 else 0,
            'avg_response_time_period': avg_response_time,
            'current_metrics': current_metrics,
            'trend': trend,
            'health_score': current_metrics['system_health'],
            'recommendations': self._generate_recommendations(current_metrics)
        }
    
    def _generate_recommendations(self, metrics: Dict[str, Any]) -> List[str]:
        """Generate recommendations based on metrics"""
        recommendations = []
        
        if metrics['success_rate'] < 0.8:
            recommendations.append('Improve query success rate by simplifying complex queries')
        
        if metrics['avg_response_time'] > 3.0:
            recommendations.append('Optimize response time by caching frequent queries')
        
        if metrics['knowledge_coverage'] < 0.8:
            recommendations.append('Expand knowledge base coverage for better answers')
        
        if not recommendations:
            recommendations.append('System performance is good. Maintain current operations.')
        
        return recommendations[:3]
    
    def reset_metrics(self):
        """Reset all metrics (for testing or restart)"""
        self.system_metrics = {
            'start_time': datetime.now(),
            'total_queries': 0,
            'successful_queries': 0,
            'failed_queries': 0,
            'active_sessions': 0,
            'peak_memory_mb': 0,
            'average_response_time': 0.0
        }
        
        for agent in self.agent_metrics:
            self.agent_metrics[agent] = {'calls': 0, 'avg_time': 0.0, 'errors': 0}
        
        self.performance_history = []
        self.alerts = []
        
        print("🔄 Monitoring metrics reset")


# Create global instance
monitoring_system = MonitoringSystem()