Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /effgen /utils /logging.py
| """ | |
| Logging utilities for effGen framework. | |
| .. note:: | |
| ``effgen.observability.logs`` is the canonical structured-logging home | |
| (``get_logger`` / ``configure_logging``). This module provides the rich | |
| console formatters and ``LoggerManager`` helpers and stays available for | |
| backwards compatibility; prefer ``effgen.observability`` in new code. | |
| This module provides comprehensive logging capabilities with rich formatting, | |
| multiple log levels, file and console handlers, structured logging, and log rotation. | |
| Features: | |
| - Rich console formatting with colors and emojis | |
| - Multiple log levels (DEBUG, INFO, WARNING, ERROR, CRITICAL) | |
| - File and console handlers with independent configurations | |
| - Structured logging with JSON output option | |
| - Automatic log rotation based on size and time | |
| - Context management for scoped logging | |
| - Performance tracking integration | |
| - Thread-safe logging operations | |
| Example: | |
| Basic usage: | |
| >>> from effgen.utils.logging import setup_logger, get_logger | |
| >>> setup_logger(level="INFO", log_dir="./logs") | |
| >>> logger = get_logger(__name__) | |
| >>> logger.info("Agent started successfully") | |
| Structured logging: | |
| >>> logger.info("Task completed", extra={ | |
| ... "task_id": "123", | |
| ... "duration": 5.2, | |
| ... "tokens_used": 1500 | |
| ... }) | |
| Context-based logging: | |
| >>> with LogContext(agent_id="agent-001"): | |
| ... logger.info("Processing task") | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import logging | |
| import logging.handlers | |
| import sys | |
| import threading | |
| import traceback | |
| from collections.abc import Callable, Iterator | |
| from contextlib import contextmanager | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from types import TracebackType | |
| from typing import Any | |
| try: | |
| from rich.console import Console # noqa: F401 | |
| from rich.logging import RichHandler | |
| from rich.traceback import install as install_rich_traceback | |
| RICH_AVAILABLE = True | |
| except ImportError: | |
| RICH_AVAILABLE = False | |
| try: | |
| from loguru import logger as loguru_logger # noqa: F401 | |
| LOGURU_AVAILABLE = True | |
| except ImportError: | |
| LOGURU_AVAILABLE = False | |
| # Thread-local storage for logging context | |
| _context_storage = threading.local() | |
| class LogLevel: | |
| """Standard log levels.""" | |
| DEBUG = "DEBUG" | |
| INFO = "INFO" | |
| WARNING = "WARNING" | |
| ERROR = "ERROR" | |
| CRITICAL = "CRITICAL" | |
| class StructuredFormatter(logging.Formatter): | |
| """ | |
| Custom formatter that outputs logs in JSON format for structured logging. | |
| This formatter converts log records to JSON format, making them easier to | |
| parse and analyze with log aggregation tools. | |
| """ | |
| def __init__(self, include_extra: bool = True) -> None: | |
| """ | |
| Initialize the structured formatter. | |
| Args: | |
| include_extra: Whether to include extra fields from log records | |
| """ | |
| super().__init__() | |
| self.include_extra = include_extra | |
| def format(self, record: logging.LogRecord) -> str: | |
| """ | |
| Format the log record as JSON. | |
| Args: | |
| record: The log record to format | |
| Returns: | |
| JSON-formatted log string | |
| """ | |
| log_data = { | |
| "timestamp": datetime.now(UTC).isoformat(), | |
| "level": record.levelname, | |
| "logger": record.name, | |
| "message": record.getMessage(), | |
| "module": record.module, | |
| "function": record.funcName, | |
| "line": record.lineno, | |
| } | |
| # Add exception info if present | |
| if record.exc_info: | |
| log_data["exception"] = { | |
| "type": record.exc_info[0].__name__, | |
| "message": str(record.exc_info[1]), | |
| "traceback": traceback.format_exception(*record.exc_info) | |
| } | |
| # Add extra fields | |
| if self.include_extra: | |
| # Get context from thread-local storage | |
| context = getattr(_context_storage, 'context', {}) | |
| if context: | |
| log_data["context"] = context | |
| # Add any extra fields from the log record | |
| for key, value in record.__dict__.items(): | |
| if key not in ['name', 'msg', 'args', 'created', 'filename', 'funcName', | |
| 'levelname', 'levelno', 'lineno', 'module', 'msecs', | |
| 'pathname', 'process', 'processName', 'relativeCreated', | |
| 'thread', 'threadName', 'exc_info', 'exc_text', 'stack_info', | |
| 'message', 'asctime']: | |
| log_data[key] = value | |
| return json.dumps(log_data) | |
| class ColoredFormatter(logging.Formatter): | |
| """ | |
| Custom formatter with color support for console output. | |
| Adds colors to log levels for better readability in console output. | |
| """ | |
| # Color codes | |
| COLORS = { | |
| 'DEBUG': '\033[36m', # Cyan | |
| 'INFO': '\033[32m', # Green | |
| 'WARNING': '\033[33m', # Yellow | |
| 'ERROR': '\033[31m', # Red | |
| 'CRITICAL': '\033[35m', # Magenta | |
| 'RESET': '\033[0m' # Reset | |
| } | |
| def __init__(self, fmt: str | None = None, use_colors: bool = True) -> None: | |
| """ | |
| Initialize the colored formatter. | |
| Args: | |
| fmt: Format string for log messages | |
| use_colors: Whether to use colors in output | |
| """ | |
| super().__init__(fmt) | |
| self.use_colors = use_colors and sys.stdout.isatty() | |
| def format(self, record: logging.LogRecord) -> str: | |
| """ | |
| Format the log record with colors. | |
| Args: | |
| record: The log record to format | |
| Returns: | |
| Formatted log string with colors | |
| """ | |
| if self.use_colors: | |
| levelname = record.levelname | |
| if levelname in self.COLORS: | |
| record.levelname = ( | |
| f"{self.COLORS[levelname]}{levelname}{self.COLORS['RESET']}" | |
| ) | |
| return super().format(record) | |
| class LoggerManager: | |
| """ | |
| Centralized logger management for the effGen framework. | |
| Manages logger instances, handlers, and configurations across the application. | |
| """ | |
| _instance = None | |
| _lock = threading.Lock() | |
| def __new__(cls) -> LoggerManager: | |
| """Singleton pattern to ensure only one logger manager exists.""" | |
| if cls._instance is None: | |
| with cls._lock: | |
| if cls._instance is None: | |
| cls._instance = super().__new__(cls) | |
| cls._instance._initialized = False | |
| return cls._instance | |
| def __init__(self) -> None: | |
| """Initialize the logger manager.""" | |
| if self._initialized: | |
| return | |
| self.loggers: dict[str, logging.Logger] = {} | |
| self.handlers: list[logging.Handler] = [] | |
| self.log_dir: Path | None = None | |
| self.level: int = logging.INFO | |
| self.structured: bool = False | |
| self._initialized = True | |
| def setup( | |
| self, | |
| level: str | int = "INFO", | |
| log_dir: str | Path | None = None, | |
| log_file: str = "effgen.log", | |
| console_output: bool = True, | |
| file_output: bool = True, | |
| structured: bool = False, | |
| use_rich: bool = True, | |
| max_file_size: int = 10 * 1024 * 1024, # 10 MB | |
| backup_count: int = 5, | |
| format_string: str | None = None, | |
| ) -> None: | |
| """ | |
| Setup logging configuration for the application. | |
| Args: | |
| level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) | |
| log_dir: Directory to store log files | |
| log_file: Name of the log file | |
| console_output: Whether to output logs to console | |
| file_output: Whether to output logs to file | |
| structured: Whether to use structured JSON logging | |
| use_rich: Whether to use rich formatting (if available) | |
| max_file_size: Maximum size of log file before rotation (bytes) | |
| backup_count: Number of backup files to keep | |
| format_string: Custom format string for log messages | |
| """ | |
| # Convert string level to int | |
| if isinstance(level, str): | |
| level = getattr(logging, level.upper(), logging.INFO) | |
| self.level = level | |
| self.structured = structured | |
| # Setup log directory | |
| if log_dir: | |
| self.log_dir = Path(log_dir) | |
| self.log_dir.mkdir(parents=True, exist_ok=True) | |
| # Clear existing handlers | |
| self._clear_handlers() | |
| # Setup format string | |
| if format_string is None: | |
| if structured: | |
| format_string = None # Will use JSON formatter | |
| else: | |
| format_string = ( | |
| "%(asctime)s - %(name)s - %(levelname)s - " | |
| "%(funcName)s:%(lineno)d - %(message)s" | |
| ) | |
| # Setup console handler | |
| if console_output: | |
| if use_rich and RICH_AVAILABLE: | |
| console_handler = RichHandler( | |
| rich_tracebacks=True, | |
| tracebacks_show_locals=True, | |
| markup=True, | |
| ) | |
| console_handler.setLevel(level) | |
| # Rich handler doesn't need a formatter | |
| else: | |
| console_handler = logging.StreamHandler(sys.stdout) | |
| console_handler.setLevel(level) | |
| if structured: | |
| formatter = StructuredFormatter() | |
| else: | |
| formatter = ColoredFormatter(format_string) | |
| console_handler.setFormatter(formatter) | |
| self.handlers.append(console_handler) | |
| # Setup file handler with rotation | |
| if file_output and self.log_dir: | |
| log_path = self.log_dir / log_file | |
| file_handler = logging.handlers.RotatingFileHandler( | |
| log_path, | |
| maxBytes=max_file_size, | |
| backupCount=backup_count, | |
| encoding='utf-8' | |
| ) | |
| file_handler.setLevel(level) | |
| if structured: | |
| formatter = StructuredFormatter() | |
| else: | |
| # File logs don't need colors | |
| formatter = logging.Formatter(format_string) | |
| file_handler.setFormatter(formatter) | |
| self.handlers.append(file_handler) | |
| # Also add a time-based rotating handler for archival | |
| if not structured: # Only add for non-structured logs | |
| time_handler = logging.handlers.TimedRotatingFileHandler( | |
| self.log_dir / f"archive_{log_file}", | |
| when='midnight', | |
| interval=1, | |
| backupCount=30, | |
| encoding='utf-8' | |
| ) | |
| time_handler.setLevel(level) | |
| time_handler.setFormatter(logging.Formatter(format_string)) | |
| self.handlers.append(time_handler) | |
| # Install rich traceback if available | |
| if use_rich and RICH_AVAILABLE: | |
| install_rich_traceback(show_locals=True) | |
| def _clear_handlers(self) -> None: | |
| """Remove all existing handlers.""" | |
| for handler in self.handlers: | |
| handler.close() | |
| self.handlers.clear() | |
| # Also clear handlers from root logger | |
| root_logger = logging.getLogger() | |
| for handler in root_logger.handlers[:]: | |
| root_logger.removeHandler(handler) | |
| def get_logger(self, name: str) -> logging.Logger: | |
| """ | |
| Get or create a logger with the given name. | |
| Args: | |
| name: Name for the logger (typically __name__) | |
| Returns: | |
| Configured logger instance | |
| """ | |
| if name in self.loggers: | |
| return self.loggers[name] | |
| logger = logging.getLogger(name) | |
| logger.setLevel(self.level) | |
| logger.handlers.clear() | |
| logger.propagate = False | |
| # Add all configured handlers | |
| for handler in self.handlers: | |
| logger.addHandler(handler) | |
| self.loggers[name] = logger | |
| return logger | |
| def set_level(self, level: str | int) -> None: | |
| """ | |
| Change the logging level for all loggers. | |
| Args: | |
| level: New logging level | |
| """ | |
| if isinstance(level, str): | |
| level = getattr(logging, level.upper(), logging.INFO) | |
| self.level = level | |
| for logger in self.loggers.values(): | |
| logger.setLevel(level) | |
| for handler in self.handlers: | |
| handler.setLevel(level) | |
| # Global logger manager instance | |
| _logger_manager = LoggerManager() | |
| def setup_logger( | |
| level: str | int = "INFO", | |
| log_dir: str | Path | None = None, | |
| log_file: str = "effgen.log", | |
| console_output: bool = True, | |
| file_output: bool = True, | |
| structured: bool = False, | |
| use_rich: bool = True, | |
| max_file_size: int = 10 * 1024 * 1024, | |
| backup_count: int = 5, | |
| format_string: str | None = None, | |
| ) -> None: | |
| """ | |
| Setup logging configuration for the application. | |
| This is the main entry point for configuring logging in effGen. | |
| Args: | |
| level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) | |
| log_dir: Directory to store log files | |
| log_file: Name of the log file | |
| console_output: Whether to output logs to console | |
| file_output: Whether to output logs to file | |
| structured: Whether to use structured JSON logging | |
| use_rich: Whether to use rich formatting (if available) | |
| max_file_size: Maximum size of log file before rotation (bytes) | |
| backup_count: Number of backup files to keep | |
| format_string: Custom format string for log messages | |
| Example: | |
| >>> setup_logger( | |
| ... level="DEBUG", | |
| ... log_dir="./logs", | |
| ... structured=True, | |
| ... use_rich=True | |
| ... ) | |
| """ | |
| _logger_manager.setup( | |
| level=level, | |
| log_dir=log_dir, | |
| log_file=log_file, | |
| console_output=console_output, | |
| file_output=file_output, | |
| structured=structured, | |
| use_rich=use_rich, | |
| max_file_size=max_file_size, | |
| backup_count=backup_count, | |
| format_string=format_string, | |
| ) | |
| def get_logger(name: str) -> logging.Logger: | |
| """ | |
| Get a logger instance for the given name. | |
| Args: | |
| name: Name for the logger (typically __name__) | |
| Returns: | |
| Configured logger instance | |
| Example: | |
| >>> logger = get_logger(__name__) | |
| >>> logger.info("Starting agent") | |
| """ | |
| return _logger_manager.get_logger(name) | |
| def set_log_level(level: str | int) -> None: | |
| """ | |
| Change the logging level for all loggers. | |
| Args: | |
| level: New logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) | |
| Example: | |
| >>> set_log_level("DEBUG") | |
| """ | |
| _logger_manager.set_level(level) | |
| def LogContext(**kwargs: Any) -> Iterator[None]: | |
| """ | |
| Context manager for adding structured logging context. | |
| Context data will be included in all log messages within the context. | |
| This is useful for tracking request IDs, agent IDs, session IDs, etc. | |
| Args: | |
| **kwargs: Key-value pairs to add to the logging context | |
| Example: | |
| >>> logger = get_logger(__name__) | |
| >>> with LogContext(agent_id="agent-001", task_id="task-123"): | |
| ... logger.info("Processing task") | |
| ... # Logs will include agent_id and task_id | |
| """ | |
| # Get or create context | |
| if not hasattr(_context_storage, 'context'): | |
| _context_storage.context = {} | |
| # Save previous context | |
| previous_context = _context_storage.context.copy() | |
| # Add new context | |
| _context_storage.context.update(kwargs) | |
| try: | |
| yield | |
| finally: | |
| # Restore previous context | |
| _context_storage.context = previous_context | |
| def log_exception( | |
| logger: logging.Logger, | |
| exception: Exception, | |
| message: str = "An error occurred", | |
| level: int = logging.ERROR, | |
| **extra_fields: Any, | |
| ) -> None: | |
| """ | |
| Log an exception with full traceback and additional context. | |
| Args: | |
| logger: Logger instance to use | |
| exception: Exception to log | |
| message: Custom message to include | |
| level: Log level to use | |
| **extra_fields: Additional fields to include in the log | |
| Example: | |
| >>> logger = get_logger(__name__) | |
| >>> try: | |
| ... raise ValueError("no such record") | |
| ... except Exception as e: | |
| ... log_exception(logger, e, "Operation failed", task_id="123") | |
| """ | |
| extra_fields['exception_type'] = type(exception).__name__ | |
| extra_fields['exception_message'] = str(exception) | |
| logger.log( | |
| level, | |
| f"{message}: {exception}", | |
| exc_info=True, | |
| extra=extra_fields | |
| ) | |
| class PerformanceLogger: | |
| """ | |
| Logger decorator for tracking function execution time. | |
| Can be used as a decorator or context manager to automatically log | |
| function execution times. | |
| """ | |
| def __init__(self, logger: logging.Logger | None = None, level: int = logging.INFO) -> None: | |
| """ | |
| Initialize the performance logger. | |
| Args: | |
| logger: Logger instance to use (creates one if not provided) | |
| level: Log level for performance logs | |
| """ | |
| self.logger = logger or get_logger(__name__) | |
| self.level = level | |
| self.start_time: float | None = None | |
| def __call__(self, func: Callable[..., Any]) -> Callable[..., Any]: | |
| """ | |
| Decorator usage. | |
| Example: | |
| >>> import time | |
| >>> logger = get_logger(__name__) | |
| >>> @PerformanceLogger(logger) | |
| ... def slow_function(): | |
| ... time.sleep(1) | |
| """ | |
| def wrapper(*args, **kwargs): | |
| import time | |
| start = time.time() | |
| try: | |
| result = func(*args, **kwargs) | |
| return result | |
| finally: | |
| duration = time.time() - start | |
| self.logger.log( | |
| self.level, | |
| f"Function '{func.__name__}' took {duration:.4f}s", | |
| extra={'function': func.__name__, 'duration': duration} | |
| ) | |
| return wrapper | |
| def __enter__(self) -> PerformanceLogger: | |
| """ | |
| Context manager entry. | |
| Example: | |
| >>> import time | |
| >>> logger = get_logger(__name__) | |
| >>> with PerformanceLogger(logger) as perf: | |
| ... time.sleep(1) | |
| """ | |
| import time | |
| self.start_time = time.time() | |
| return self | |
| def __exit__( | |
| self, | |
| exc_type: type[BaseException] | None, | |
| exc_val: BaseException | None, | |
| exc_tb: TracebackType | None, | |
| ) -> None: | |
| """Context manager exit.""" | |
| import time | |
| if self.start_time: | |
| duration = time.time() - self.start_time | |
| self.logger.log( | |
| self.level, | |
| f"Code block took {duration:.4f}s", | |
| extra={'duration': duration} | |
| ) | |
| # Convenience functions for common logging patterns | |
| def log_agent_start(logger: logging.Logger, agent_id: str, **metadata: Any) -> None: | |
| """Log agent startup with metadata. | |
| Args: | |
| logger: Where the line is written. | |
| agent_id: The agent that started. | |
| **metadata: Extra fields recorded with the entry. | |
| """ | |
| logger.info( | |
| f"Agent started: {agent_id}", | |
| extra={'event': 'agent_start', 'agent_id': agent_id, **metadata} | |
| ) | |
| def log_agent_stop(logger: logging.Logger, agent_id: str, **metadata: Any) -> None: | |
| """Log agent shutdown with metadata. | |
| Args: | |
| logger: Where the line is written. | |
| agent_id: The agent that stopped. | |
| **metadata: Extra fields recorded with the entry. | |
| """ | |
| logger.info( | |
| f"Agent stopped: {agent_id}", | |
| extra={'event': 'agent_stop', 'agent_id': agent_id, **metadata} | |
| ) | |
| def log_task_start(logger: logging.Logger, task_id: str, **metadata: Any) -> None: | |
| """Log task start with metadata. | |
| Args: | |
| logger: Where the line is written. | |
| task_id: The task that started. | |
| **metadata: Extra fields recorded with the entry. | |
| """ | |
| logger.info( | |
| f"Task started: {task_id}", | |
| extra={'event': 'task_start', 'task_id': task_id, **metadata} | |
| ) | |
| def log_task_complete(logger: logging.Logger, task_id: str, **metadata: Any) -> None: | |
| """Log task completion with metadata. | |
| Args: | |
| logger: Where the line is written. | |
| task_id: The task that finished. | |
| **metadata: Extra fields recorded with the entry. | |
| """ | |
| logger.info( | |
| f"Task completed: {task_id}", | |
| extra={'event': 'task_complete', 'task_id': task_id, **metadata} | |
| ) | |
| def log_task_error(logger: logging.Logger, task_id: str, error: Exception, **metadata: Any) -> None: | |
| """Log task error with metadata. | |
| Args: | |
| logger: Where the line is written. | |
| task_id: The task that failed. | |
| error: The exception that ended it. | |
| **metadata: Extra fields recorded with the entry. | |
| """ | |
| log_exception( | |
| logger, | |
| error, | |
| f"Task failed: {task_id}", | |
| event='task_error', | |
| task_id=task_id, | |
| **metadata | |
| ) | |
| def log_tool_call(logger: logging.Logger, tool_name: str, **metadata: Any) -> None: | |
| """Log tool invocation with metadata. | |
| Args: | |
| logger: Where the line is written. | |
| tool_name: The tool that was called. | |
| **metadata: Extra fields recorded with the entry. | |
| """ | |
| logger.debug( | |
| f"Tool called: {tool_name}", | |
| extra={'event': 'tool_call', 'tool_name': tool_name, **metadata} | |
| ) | |
| def log_model_inference(logger: logging.Logger, model_name: str, **metadata: Any) -> None: | |
| """Log model inference with metadata. | |
| Args: | |
| logger: Where the line is written. | |
| model_name: The model that was called. | |
| **metadata: Extra fields recorded with the entry. | |
| """ | |
| logger.debug( | |
| f"Model inference: {model_name}", | |
| extra={'event': 'model_inference', 'model_name': model_name, **metadata} | |
| ) | |
| __all__ = [ | |
| 'LogLevel', | |
| 'StructuredFormatter', | |
| 'ColoredFormatter', | |
| 'LoggerManager', | |
| 'setup_logger', | |
| 'get_logger', | |
| 'set_log_level', | |
| 'LogContext', | |
| 'log_exception', | |
| 'PerformanceLogger', | |
| 'log_agent_start', | |
| 'log_agent_stop', | |
| 'log_task_start', | |
| 'log_task_complete', | |
| 'log_task_error', | |
| 'log_tool_call', | |
| 'log_model_inference', | |
| ] | |
Xet Storage Details
- Size:
- 22.8 kB
- Xet hash:
- 325ed8a94e939680c646749526d29e42200213c96468b196e02e674629c1b99d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.