Spaces:

dn-scribe
/

attstore

Runtime error

attstore / logging_utils.py

feat: limit log retention

f27c4a0 11 months ago

7.95 kB

import logging
import socket
from datetime import datetime, timedelta
from flask import request, current_app
from models import SystemLog
from app import db

# Create logger
logger = logging.getLogger(__name__)

def get_logger(name):
    """Get a logger instance"""
    return logging.getLogger(name)

def log_event(severity, message, event_type=None, user_id=None, source_ip=None, 
              facility=None, app_name=None, proc_id=None, msg_id=None, structured_data=None):
    """
    Log an event in RFC 5424 format to the database
    
    Severity levels (RFC 5424):
    0 - Emergency, 1 - Alert, 2 - Critical, 3 - Error
    4 - Warning, 5 - Notice, 6 - Informational, 7 - Debug
    """
    try:
        config = current_app.config.get('STORAGE_CONFIG')
        
        # Default values
        if facility is None:
            facility = config.SYSLOG_FACILITY if config else 16
        if app_name is None:
            app_name = config.SYSLOG_APP_NAME if config else 'attestation-store'
        if source_ip is None and request:
            source_ip = request.remote_addr
        
        hostname = config.SYSLOG_HOSTNAME if config else socket.gethostname()
        
        # Create log entry
        log_entry = SystemLog(
            facility=facility,
            severity=severity,
            version=1,
            timestamp=datetime.utcnow(),
            hostname=hostname,
            app_name=app_name,
            proc_id=proc_id,
            msg_id=msg_id,
            structured_data=structured_data,
            message=message,
            user_id=user_id,
            source_ip=source_ip,
            event_type=event_type
        )
        
        db.session.add(log_entry)
        db.session.commit()
        
        # Also log to application logger
        log_level = {
            0: logging.CRITICAL,  # Emergency
            1: logging.CRITICAL,  # Alert
            2: logging.CRITICAL,  # Critical
            3: logging.ERROR,     # Error
            4: logging.WARNING,   # Warning
            5: logging.INFO,      # Notice
            6: logging.INFO,      # Informational
            7: logging.DEBUG      # Debug
        }.get(severity, logging.INFO)
        
        logger.log(log_level, f"[{event_type}] {message}")
        
    except Exception as e:
        # Fallback to standard logging if database logging fails
        logger.error(f"Failed to log event to database: {e}")
        logger.log(logging.INFO, f"[{event_type}] {message}")

def parse_rfc5424_log(log_message):
    """
    Parse an RFC 5424 syslog message
    Returns a dictionary with parsed fields
    """
    try:
        # Basic RFC 5424 parsing
        # Format: <priority>version timestamp hostname app-name procid msgid structured-data msg
        
        if not log_message.startswith('<'):
            raise ValueError("Invalid RFC 5424 format - missing priority")
        
        # Extract priority
        priority_end = log_message.find('>')
        if priority_end == -1:
            raise ValueError("Invalid RFC 5424 format - missing priority end")
        
        priority = int(log_message[1:priority_end])
        facility = priority // 8
        severity = priority % 8
        
        # Split remaining parts
        remaining = log_message[priority_end + 1:].strip()
        parts = remaining.split(' ', 6)
        
        if len(parts) < 6:
            raise ValueError("Invalid RFC 5424 format - insufficient parts")
        
        version = int(parts[0]) if parts[0] != '-' else 1
        timestamp_str = parts[1] if parts[1] != '-' else None
        hostname = parts[2] if parts[2] != '-' else None
        app_name = parts[3] if parts[3] != '-' else None
        proc_id = parts[4] if parts[4] != '-' else None
        msg_id = parts[5] if parts[5] != '-' else None
        
        # Parse structured data and message
        structured_data = None
        message = ""
        
        if len(parts) > 6:
            remainder = parts[6]
            if remainder.startswith('['):
                # Find end of structured data
                bracket_count = 0
                sd_end = 0
                for i, char in enumerate(remainder):
                    if char == '[':
                        bracket_count += 1
                    elif char == ']':
                        bracket_count -= 1
                        if bracket_count == 0:
                            sd_end = i + 1
                            break
                
                if sd_end > 0:
                    structured_data = remainder[:sd_end]
                    message = remainder[sd_end:].strip()
                else:
                    message = remainder
            else:
                if remainder != '-':
                    message = remainder
        
        # Parse timestamp
        timestamp = None
        if timestamp_str:
            try:
                # Handle various timestamp formats
                for fmt in ['%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S']:
                    try:
                        timestamp = datetime.strptime(timestamp_str, fmt)
                        break
                    except ValueError:
                        continue
            except:
                timestamp = datetime.utcnow()
        
        return {
            'facility': facility,
            'severity': severity,
            'version': version,
            'timestamp': timestamp or datetime.utcnow(),
            'hostname': hostname,
            'app_name': app_name,
            'proc_id': proc_id,
            'msg_id': msg_id,
            'structured_data': structured_data,
            'message': message
        }
        
    except Exception as e:
        logger.error(f"Failed to parse RFC 5424 log: {e}")
        # Return basic log entry for unparseable messages
        return {
            'facility': 16,  # Local use
            'severity': 6,   # Info
            'version': 1,
            'timestamp': datetime.utcnow(),
            'hostname': None,
            'app_name': 'unknown',
            'proc_id': None,
            'msg_id': None,
            'structured_data': None,
            'message': log_message
        }

def cleanup_old_logs(retention_days=None):
    """
    Delete logs older than the specified retention period.
    
    Args:
        retention_days (int, optional): Number of days to retain logs.
            If not provided, uses LOG_RETENTION_DAYS from config.
            Set to 0 to skip cleanup.
    
    Returns:
        int: Number of log entries deleted
    
    Example:
        # Delete logs older than 10 days (using default config)
        deleted_count = cleanup_old_logs()
        
        # Delete logs older than 30 days
        deleted_count = cleanup_old_logs(retention_days=30)
    """
    try:
        # Get retention period from config if not specified
        if retention_days is None:
            config = current_app.config.get('STORAGE_CONFIG')
            retention_days = config.LOG_RETENTION_DAYS if config else 10
        
        # Skip cleanup if retention is 0 or negative
        if retention_days <= 0:
            logger.info("Log cleanup skipped (retention disabled)")
            return 0
        
        # Calculate cutoff date
        cutoff_date = datetime.utcnow() - timedelta(days=retention_days)
        
        # Delete old logs
        deleted = SystemLog.query.filter(SystemLog.timestamp < cutoff_date).delete()
        db.session.commit()
        
        if deleted > 0:
            logger.info(f"Deleted {deleted} log entries older than {retention_days} days (before {cutoff_date.isoformat()})")
        else:
            logger.debug(f"No logs found older than {retention_days} days")
        
        return deleted
        
    except Exception as e:
        logger.error(f"Failed to cleanup old logs: {e}")
        db.session.rollback()
        return 0

	import logging
	import socket
	from datetime import datetime, timedelta
	from flask import request, current_app
	from models import SystemLog
	from app import db

	# Create logger
	logger = logging.getLogger(__name__)

	def get_logger(name):
	"""Get a logger instance"""
	return logging.getLogger(name)

	def log_event(severity, message, event_type=None, user_id=None, source_ip=None,
	facility=None, app_name=None, proc_id=None, msg_id=None, structured_data=None):
	"""
	Log an event in RFC 5424 format to the database

	Severity levels (RFC 5424):
	0 - Emergency, 1 - Alert, 2 - Critical, 3 - Error
	4 - Warning, 5 - Notice, 6 - Informational, 7 - Debug
	"""
	try:
	config = current_app.config.get('STORAGE_CONFIG')

	# Default values
	if facility is None:
	facility = config.SYSLOG_FACILITY if config else 16
	if app_name is None:
	app_name = config.SYSLOG_APP_NAME if config else 'attestation-store'
	if source_ip is None and request:
	source_ip = request.remote_addr

	hostname = config.SYSLOG_HOSTNAME if config else socket.gethostname()

	# Create log entry
	log_entry = SystemLog(
	facility=facility,
	severity=severity,
	version=1,
	timestamp=datetime.utcnow(),
	hostname=hostname,
	app_name=app_name,
	proc_id=proc_id,
	msg_id=msg_id,
	structured_data=structured_data,
	message=message,
	user_id=user_id,
	source_ip=source_ip,
	event_type=event_type
	)

	db.session.add(log_entry)
	db.session.commit()

	# Also log to application logger
	log_level = {
	0: logging.CRITICAL, # Emergency
	1: logging.CRITICAL, # Alert
	2: logging.CRITICAL, # Critical
	3: logging.ERROR, # Error
	4: logging.WARNING, # Warning
	5: logging.INFO, # Notice
	6: logging.INFO, # Informational
	7: logging.DEBUG # Debug
	}.get(severity, logging.INFO)

	logger.log(log_level, f"[{event_type}] {message}")

	except Exception as e:
	# Fallback to standard logging if database logging fails
	logger.error(f"Failed to log event to database: {e}")
	logger.log(logging.INFO, f"[{event_type}] {message}")

	def parse_rfc5424_log(log_message):
	"""
	Parse an RFC 5424 syslog message
	Returns a dictionary with parsed fields
	"""
	try:
	# Basic RFC 5424 parsing
	# Format: <priority>version timestamp hostname app-name procid msgid structured-data msg

	if not log_message.startswith('<'):
	raise ValueError("Invalid RFC 5424 format - missing priority")

	# Extract priority
	priority_end = log_message.find('>')
	if priority_end == -1:
	raise ValueError("Invalid RFC 5424 format - missing priority end")

	priority = int(log_message[1:priority_end])
	facility = priority // 8
	severity = priority % 8

	# Split remaining parts
	remaining = log_message[priority_end + 1:].strip()
	parts = remaining.split(' ', 6)

	if len(parts) < 6:
	raise ValueError("Invalid RFC 5424 format - insufficient parts")

	version = int(parts[0]) if parts[0] != '-' else 1
	timestamp_str = parts[1] if parts[1] != '-' else None
	hostname = parts[2] if parts[2] != '-' else None
	app_name = parts[3] if parts[3] != '-' else None
	proc_id = parts[4] if parts[4] != '-' else None
	msg_id = parts[5] if parts[5] != '-' else None

	# Parse structured data and message
	structured_data = None
	message = ""

	if len(parts) > 6:
	remainder = parts[6]
	if remainder.startswith('['):
	# Find end of structured data
	bracket_count = 0
	sd_end = 0
	for i, char in enumerate(remainder):
	if char == '[':
	bracket_count += 1
	elif char == ']':
	bracket_count -= 1
	if bracket_count == 0:
	sd_end = i + 1
	break

	if sd_end > 0:
	structured_data = remainder[:sd_end]
	message = remainder[sd_end:].strip()
	else:
	message = remainder
	else:
	if remainder != '-':
	message = remainder

	# Parse timestamp
	timestamp = None
	if timestamp_str:
	try:
	# Handle various timestamp formats
	for fmt in ['%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S']:
	try:
	timestamp = datetime.strptime(timestamp_str, fmt)
	break
	except ValueError:
	continue
	except:
	timestamp = datetime.utcnow()

	return {
	'facility': facility,
	'severity': severity,
	'version': version,
	'timestamp': timestamp or datetime.utcnow(),
	'hostname': hostname,
	'app_name': app_name,
	'proc_id': proc_id,
	'msg_id': msg_id,
	'structured_data': structured_data,
	'message': message
	}

	except Exception as e:
	logger.error(f"Failed to parse RFC 5424 log: {e}")
	# Return basic log entry for unparseable messages
	return {
	'facility': 16, # Local use
	'severity': 6, # Info
	'version': 1,
	'timestamp': datetime.utcnow(),
	'hostname': None,
	'app_name': 'unknown',
	'proc_id': None,
	'msg_id': None,
	'structured_data': None,
	'message': log_message
	}

	def cleanup_old_logs(retention_days=None):
	"""
	Delete logs older than the specified retention period.

	Args:
	retention_days (int, optional): Number of days to retain logs.
	If not provided, uses LOG_RETENTION_DAYS from config.
	Set to 0 to skip cleanup.

	Returns:
	int: Number of log entries deleted

	Example:
	# Delete logs older than 10 days (using default config)
	deleted_count = cleanup_old_logs()

	# Delete logs older than 30 days
	deleted_count = cleanup_old_logs(retention_days=30)
	"""
	try:
	# Get retention period from config if not specified
	if retention_days is None:
	config = current_app.config.get('STORAGE_CONFIG')
	retention_days = config.LOG_RETENTION_DAYS if config else 10

	# Skip cleanup if retention is 0 or negative
	if retention_days <= 0:
	logger.info("Log cleanup skipped (retention disabled)")
	return 0

	# Calculate cutoff date
	cutoff_date = datetime.utcnow() - timedelta(days=retention_days)

	# Delete old logs
	deleted = SystemLog.query.filter(SystemLog.timestamp < cutoff_date).delete()
	db.session.commit()

	if deleted > 0:
	logger.info(f"Deleted {deleted} log entries older than {retention_days} days (before {cutoff_date.isoformat()})")
	else:
	logger.debug(f"No logs found older than {retention_days} days")

	return deleted

	except Exception as e:
	logger.error(f"Failed to cleanup old logs: {e}")
	db.session.rollback()
	return 0