Download core/utils/logger.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/logger.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/utils/logger.py
-
curl -L -o logger.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/logger.py
10 kB
| import logging | |
| import os | |
| import socket | |
| import structlog | |
| # Load environment variables from .env file if it exists | |
| try: | |
| from dotenv import load_dotenv | |
| from pathlib import Path | |
| env_file = Path(__file__).parent.parent.parent / ".env" | |
| if env_file.exists(): | |
| load_dotenv(env_file, override=True) | |
| except ImportError: | |
| pass # dotenv not available, use system env vars only | |
| ENV_MODE = os.getenv("ENV_MODE", "LOCAL").upper() | |
| # Set default logging level based on environment | |
| # Production should be INFO (less verbose), local/staging can be DEBUG | |
| default_level = "INFO" if ENV_MODE == "PRODUCTION" else "DEBUG" | |
| LOGGING_LEVEL = logging.getLevelNamesMapping().get( | |
| os.getenv("LOGGING_LEVEL", default_level).upper(), | |
| logging.INFO, | |
| ) | |
| # Set root logger level (but don't add default handler - we'll add our own) | |
| logging.getLogger().setLevel(LOGGING_LEVEL) | |
| # Clear any existing handlers to avoid duplicate output | |
| logging.getLogger().handlers.clear() | |
| # Keep third-party library noise down (only warnings+) | |
| NOISY_LOGGERS = ( | |
| # AWS SDK | |
| "boto3", "botocore", "urllib3", "s3transfer", "watchtower", | |
| # HTTP clients | |
| "httpcore", "httpx", "aiohttp", "requests", "hpack", | |
| # Async / event loop | |
| "asyncio", "concurrent", | |
| # AI SDKs | |
| "openai", "anthropic", "httpx._client", | |
| # Web frameworks | |
| "uvicorn.access", "uvicorn.error", "fastapi", | |
| # Database | |
| "sqlalchemy", "databases", "aiosqlite", | |
| # Other | |
| "websockets", "multipart", "charset_normalizer", | |
| ) | |
| for noisy_logger in NOISY_LOGGERS: | |
| logging.getLogger(noisy_logger).setLevel(logging.WARNING) | |
| # Optional: separate CloudWatch handler level (defaults to INFO) | |
| CLOUDWATCH_LOG_LEVEL = logging.getLevelNamesMapping().get( | |
| os.getenv("CLOUDWATCH_LOG_LEVEL", "INFO").upper(), | |
| LOGGING_LEVEL, | |
| ) | |
| # Common pre-chain used by both console and CloudWatch formatters | |
| # Simplified: only func_name for cleaner output (filename:lineno adds clutter) | |
| foreign_pre_chain = [ | |
| structlog.stdlib.add_log_level, | |
| structlog.stdlib.PositionalArgumentsFormatter(), | |
| structlog.processors.TimeStamper(fmt="iso"), | |
| structlog.processors.CallsiteParameterAdder( | |
| { | |
| structlog.processors.CallsiteParameter.FILENAME, | |
| structlog.processors.CallsiteParameter.FUNC_NAME, | |
| structlog.processors.CallsiteParameter.LINENO, | |
| } | |
| ), | |
| structlog.contextvars.merge_contextvars, | |
| ] | |
| # Renderer selection per environment | |
| if ENV_MODE in ("LOCAL", "STAGING"): | |
| console_renderer = structlog.dev.ConsoleRenderer(colors=True) | |
| else: | |
| console_renderer = structlog.processors.JSONRenderer() | |
| # Configure structlog to emit into stdlib logging | |
| structlog.configure( | |
| processors=[ | |
| structlog.stdlib.add_log_level, | |
| structlog.stdlib.PositionalArgumentsFormatter(), | |
| structlog.processors.TimeStamper(fmt="iso"), | |
| structlog.processors.CallsiteParameterAdder( | |
| { | |
| structlog.processors.CallsiteParameter.FILENAME, | |
| structlog.processors.CallsiteParameter.FUNC_NAME, | |
| structlog.processors.CallsiteParameter.LINENO, | |
| } | |
| ), | |
| structlog.contextvars.merge_contextvars, | |
| structlog.stdlib.ProcessorFormatter.wrap_for_formatter, | |
| ], | |
| logger_factory=structlog.stdlib.LoggerFactory(), | |
| wrapper_class=structlog.stdlib.BoundLogger, | |
| cache_logger_on_first_use=True, | |
| ) | |
| # Console handler (stdout) | |
| console_handler = logging.StreamHandler() | |
| console_handler.setLevel(LOGGING_LEVEL) | |
| console_handler.setFormatter( | |
| structlog.stdlib.ProcessorFormatter( | |
| processor=console_renderer, | |
| foreign_pre_chain=foreign_pre_chain, | |
| ) | |
| ) | |
| logging.getLogger().addHandler(console_handler) | |
| def _setup_cloudwatch_logging() -> None: | |
| """Attach a CloudWatch handler when enabled and available.""" | |
| if ENV_MODE not in ("PRODUCTION", "STAGING"): | |
| return | |
| default_enabled = "true" if ENV_MODE == "PRODUCTION" else "false" | |
| cloudwatch_enabled = ( | |
| os.getenv("CLOUDWATCH_LOGGING_ENABLED", default_enabled).lower() == "true" | |
| ) | |
| if not cloudwatch_enabled: | |
| return | |
| try: | |
| import watchtower | |
| import boto3 | |
| default_log_group = ( | |
| "/kortix/production" if ENV_MODE == "PRODUCTION" else "/kortix/staging" | |
| ) | |
| log_group_name = os.getenv("CLOUDWATCH_LOG_GROUP", default_log_group) | |
| aws_region = os.getenv("AWS_DEFAULT_REGION", os.getenv("AWS_REGION", "us-west-2")) | |
| log_stream_name = os.getenv("CLOUDWATCH_LOG_STREAM") | |
| if not log_stream_name: | |
| log_stream_name = f"api/{socket.gethostname()}" | |
| aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID") | |
| aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY") | |
| if aws_access_key_id and aws_secret_access_key: | |
| logs_client = boto3.client( | |
| "logs", | |
| region_name=aws_region, | |
| aws_access_key_id=aws_access_key_id, | |
| aws_secret_access_key=aws_secret_access_key, | |
| ) | |
| else: | |
| logs_client = boto3.client("logs", region_name=aws_region) | |
| # Best-effort: ensure log group exists | |
| try: | |
| logs_client.create_log_group(logGroupName=log_group_name) | |
| except logs_client.exceptions.ResourceAlreadyExistsException: | |
| pass | |
| except Exception as group_error: | |
| logging.getLogger().warning( | |
| f"Could not create CloudWatch log group '{log_group_name}': {group_error}" | |
| ) | |
| cloudwatch_handler = watchtower.CloudWatchLogHandler( | |
| log_group=log_group_name, | |
| stream_name=log_stream_name, | |
| use_queues=True, | |
| send_interval=5, | |
| max_batch_size=100, | |
| max_batch_count=1000, | |
| boto3_client=logs_client, | |
| ) | |
| cloudwatch_handler.setLevel(CLOUDWATCH_LOG_LEVEL) | |
| cloudwatch_handler.setFormatter( | |
| structlog.stdlib.ProcessorFormatter( | |
| processor=structlog.processors.JSONRenderer(), | |
| foreign_pre_chain=foreign_pre_chain, | |
| ) | |
| ) | |
| logging.getLogger().addHandler(cloudwatch_handler) | |
| logging.getLogger().info( | |
| f"CloudWatch logging enabled: group={log_group_name}, stream={log_stream_name}" | |
| ) | |
| except ImportError: | |
| logging.getLogger().warning( | |
| "watchtower not installed. CloudWatch logging disabled. Install with: uv add watchtower" | |
| ) | |
| except Exception as e: | |
| logging.getLogger().warning( | |
| f"Failed to setup CloudWatch logging: {e}. Falling back to stdout only." | |
| ) | |
| _setup_cloudwatch_logging() | |
| logger: structlog.stdlib.BoundLogger = structlog.get_logger() | |
| from pathlib import Path | |
| from datetime import datetime | |
| from typing import Optional, Any | |
| import time | |
| class FileDebugLogger: | |
| _instance = None | |
| _files: dict = {} | |
| _start_times: dict = {} | |
| _debug_dir: Path = None | |
| def __new__(cls): | |
| if cls._instance is None: | |
| cls._instance = super().__new__(cls) | |
| cls._instance._files = {} | |
| cls._instance._start_times = {} | |
| cls._instance._debug_dir = Path(__file__).parent.parent.parent / "debug_logs" | |
| return cls._instance | |
| def _get_file(self, session: str, category: str): | |
| key = f"{session}:{category}" | |
| if key not in self._files: | |
| self._debug_dir.mkdir(exist_ok=True) | |
| timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") | |
| filename = f"{category}_{session[:8]}_{timestamp}.log" if session else f"{category}_{timestamp}.log" | |
| filepath = self._debug_dir / filename | |
| self._files[key] = open(filepath, 'a', encoding='utf-8') | |
| self._start_times[key] = time.perf_counter() | |
| self._files[key].write(f"# Debug Log: {category}\n") | |
| self._files[key].write(f"# Session: {session or 'default'}\n") | |
| self._files[key].write(f"# Started: {datetime.now().isoformat()}\n") | |
| self._files[key].write(f"# {'=' * 60}\n\n") | |
| self._files[key].flush() | |
| return self._files[key], self._start_times[key] | |
| def log( | |
| self, | |
| message: str, | |
| session: str = "default", | |
| category: str = "debug", | |
| save_to_file: bool = True, | |
| **kwargs | |
| ): | |
| if not save_to_file: | |
| return | |
| try: | |
| file, start_time = self._get_file(session, category) | |
| elapsed_ms = (time.perf_counter() - start_time) * 1000 | |
| extras = " | ".join(f"{k}={v}" for k, v in kwargs.items()) if kwargs else "" | |
| extras_str = f" | {extras}" if extras else "" | |
| line = f"[{elapsed_ms:10.1f}ms] {message}{extras_str}\n" | |
| file.write(line) | |
| file.flush() | |
| except Exception as e: | |
| pass | |
| def close(self, session: str = None, category: str = None): | |
| if session and category: | |
| key = f"{session}:{category}" | |
| if key in self._files: | |
| try: | |
| self._files[key].close() | |
| except: | |
| pass | |
| del self._files[key] | |
| if key in self._start_times: | |
| del self._start_times[key] | |
| else: | |
| for f in self._files.values(): | |
| try: | |
| f.close() | |
| except: | |
| pass | |
| self._files.clear() | |
| self._start_times.clear() | |
| def close_session(self, session: str): | |
| keys_to_remove = [k for k in self._files.keys() if k.startswith(f"{session}:")] | |
| for key in keys_to_remove: | |
| try: | |
| self._files[key].close() | |
| except: | |
| pass | |
| del self._files[key] | |
| if key in self._start_times: | |
| del self._start_times[key] | |
| file_debug = FileDebugLogger() | |