# Copyright (C) 2026 Mattia Zatelli # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . """ Background tasks for tiered storage system Handles automatic demotion jobs and maintenance tasks """ import logging import asyncio from typing import Dict, Any from datetime import datetime, timezone from contextlib import asynccontextmanager from app.core.config import get_settings from app.services.tiering import tiering_service from app.services.metrics import metrics_emitter logger = logging.getLogger(__name__) settings = get_settings() class BackgroundTaskManager: """Manager for background tasks related to tiered storage""" def __init__(self): self.settings = get_settings() self.running_tasks = {} self.task_stats = { "demotion_job": { "last_run": None, "last_duration_seconds": 0, "total_runs": 0, "success_count": 0, "error_count": 0 } } async def start_background_tasks(self): """Start all background tasks""" logger.info("Starting tiered storage background tasks") # Start demotion job scheduler if self.settings.ENABLE_AUTO_DEMOTION: demotion_task = asyncio.create_task(self._run_demotion_scheduler()) self.running_tasks["demotion_scheduler"] = demotion_task # Start metrics cleanup task cleanup_task = asyncio.create_task(self._run_metrics_cleanup()) self.running_tasks["metrics_cleanup"] = cleanup_task logger.info(f"Started {len(self.running_tasks)} background tasks") async def stop_background_tasks(self): """Stop all background tasks""" logger.info("Stopping tiered storage background tasks") for task_name, task in self.running_tasks.items(): if not task.done(): logger.info(f"Cancelling task: {task_name}") task.cancel() try: await task except asyncio.CancelledError: logger.info(f"Task cancelled: {task_name}") self.running_tasks.clear() logger.info("All background tasks stopped") async def _run_demotion_scheduler(self): """Run the demotion job on a schedule""" interval_seconds = self.settings.DEMOTION_INTERVAL_SECONDS logger.info(f"Starting demotion scheduler with {interval_seconds}s interval") while True: try: # Wait for the next scheduled run await asyncio.sleep(interval_seconds) # Run demotion job await self._run_demotion_job() except asyncio.CancelledError: logger.info("Demotion scheduler cancelled") break except Exception as e: logger.error(f"Error in demotion scheduler: {e}") # Continue running despite errors await asyncio.sleep(60) # Wait 1 minute before retrying async def _run_demotion_job(self): """Execute a single demotion job run""" job_start = datetime.now(timezone.utc) logger.info("Starting automatic demotion job") try: # Track the job execution with metrics if metrics_emitter: async with metrics_emitter.track_operation( "automatic_demotion_job", 0, # No specific project ID for batch jobs {"trigger": "scheduled"} ): # Run the demotion job stats = await tiering_service.demote_inactive_projects( batch_size=self.settings.TIERING_BATCH_SIZE ) # Update task stats job_end = datetime.now(timezone.utc) duration = (job_end - job_start).total_seconds() self.task_stats["demotion_job"].update({ "last_run": job_end.isoformat(), "last_duration_seconds": duration, "total_runs": self.task_stats["demotion_job"]["total_runs"] + 1, "success_count": self.task_stats["demotion_job"]["success_count"] + 1 }) logger.info(f"Demotion job completed successfully: {stats}") else: # Fallback without metrics stats = await tiering_service.demote_inactive_projects( batch_size=self.settings.TIERING_BATCH_SIZE ) job_end = datetime.now(timezone.utc) duration = (job_end - job_start).total_seconds() self.task_stats["demotion_job"].update({ "last_run": job_end.isoformat(), "last_duration_seconds": duration, "total_runs": self.task_stats["demotion_job"]["total_runs"] + 1, "success_count": self.task_stats["demotion_job"]["success_count"] + 1 }) logger.info(f"Demotion job completed: {stats}") except Exception as e: # Update error stats self.task_stats["demotion_job"]["error_count"] += 1 logger.error(f"Demotion job failed: {e}") # Emit error metric if metrics_emitter: await metrics_emitter._emit_operation_metric( operation_type="automatic_demotion_job", status="failure", duration_ms=0, project_id=0, error=str(e) ) async def _run_metrics_cleanup(self): """Clean up old metrics data""" cleanup_interval = 24 * 3600 # 24 hours logger.info(f"Starting metrics cleanup task with {cleanup_interval}s interval") while True: try: await asyncio.sleep(cleanup_interval) logger.info("Running metrics cleanup") # Clean up old metric events from Redis from app.core.redis import RedisCache # Clean events older than 30 days cutoff_timestamp = int((datetime.now(timezone.utc).timestamp() - (30 * 24 * 3600))) # Get list of metric keys and clean old ones pattern = "tiered_storage:events:*" keys = await RedisCache._redis.keys(pattern) deleted_count = 0 for key in keys: key_str = key.decode() if isinstance(key, bytes) else key timestamp_part = key_str.split(':')[-1] try: key_timestamp = int(timestamp_part) if key_timestamp < cutoff_timestamp: await RedisCache.delete(key_str) deleted_count += 1 except (ValueError, IndexError): # Skip malformed keys continue # Trim the operation events list to keep only recent events list_key = "tiered_storage:operation_events" await RedisCache.ltrim(list_key, 0, 5000) # Keep last 5k events logger.info(f"Metrics cleanup completed: deleted {deleted_count} old events") except asyncio.CancelledError: logger.info("Metrics cleanup task cancelled") break except Exception as e: logger.error(f"Error in metrics cleanup: {e}") await asyncio.sleep(3600) # Wait 1 hour before retrying async def get_task_status(self) -> Dict[str, Any]: """Get status of background tasks""" status = { "tasks": {}, "statistics": self.task_stats, "timestamp": datetime.now(timezone.utc).isoformat() } for task_name, task in self.running_tasks.items(): status["tasks"][task_name] = { "running": not task.done(), "cancelled": task.cancelled() if task.done() else False, "exception": str(task.exception()) if task.done() and task.exception() else None } return status async def trigger_manual_demotion(self, batch_size: int = None) -> Dict[str, Any]: """Manually trigger a demotion job""" logger.info("Manual demotion job triggered") try: stats = await tiering_service.demote_inactive_projects(batch_size) # Update manual trigger stats if "manual_triggers" not in self.task_stats: self.task_stats["manual_triggers"] = {"count": 0, "last_run": None} self.task_stats["manual_triggers"]["count"] += 1 self.task_stats["manual_triggers"]["last_run"] = datetime.now(timezone.utc).isoformat() return { "status": "completed", "stats": stats, "timestamp": datetime.now(timezone.utc).isoformat() } except Exception as e: logger.error(f"Manual demotion job failed: {e}") return { "status": "failed", "error": str(e), "timestamp": datetime.now(timezone.utc).isoformat() } # Global instance background_task_manager = BackgroundTaskManager() @asynccontextmanager async def lifespan_manager(): """Context manager for application lifespan - starts/stops background tasks""" try: # Start background tasks await background_task_manager.start_background_tasks() yield finally: # Stop background tasks await background_task_manager.stop_background_tasks()