bimquery-api / app /services /background_tasks.py
BIM Query Deploy
Claude Opus 4.7 (1M context)
deploy: initial BIM Query API on Hugging Face Spaces
e6be672
Raw History Blame Contribute Delete
10.4 kB
# Copyright (C) 2026 Mattia Zatelli
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Background tasks for tiered storage system
Handles automatic demotion jobs and maintenance tasks
"""
import logging
import asyncio
from typing import Dict, Any
from datetime import datetime, timezone
from contextlib import asynccontextmanager
from app.core.config import get_settings
from app.services.tiering import tiering_service
from app.services.metrics import metrics_emitter
logger = logging.getLogger(__name__)
settings = get_settings()
class BackgroundTaskManager:
"""Manager for background tasks related to tiered storage"""
def __init__(self):
self.settings = get_settings()
self.running_tasks = {}
self.task_stats = {
"demotion_job": {
"last_run": None,
"last_duration_seconds": 0,
"total_runs": 0,
"success_count": 0,
"error_count": 0
}
}
async def start_background_tasks(self):
"""Start all background tasks"""
logger.info("Starting tiered storage background tasks")
# Start demotion job scheduler
if self.settings.ENABLE_AUTO_DEMOTION:
demotion_task = asyncio.create_task(self._run_demotion_scheduler())
self.running_tasks["demotion_scheduler"] = demotion_task
# Start metrics cleanup task
cleanup_task = asyncio.create_task(self._run_metrics_cleanup())
self.running_tasks["metrics_cleanup"] = cleanup_task
logger.info(f"Started {len(self.running_tasks)} background tasks")
async def stop_background_tasks(self):
"""Stop all background tasks"""
logger.info("Stopping tiered storage background tasks")
for task_name, task in self.running_tasks.items():
if not task.done():
logger.info(f"Cancelling task: {task_name}")
task.cancel()
try:
await task
except asyncio.CancelledError:
logger.info(f"Task cancelled: {task_name}")
self.running_tasks.clear()
logger.info("All background tasks stopped")
async def _run_demotion_scheduler(self):
"""Run the demotion job on a schedule"""
interval_seconds = self.settings.DEMOTION_INTERVAL_SECONDS
logger.info(f"Starting demotion scheduler with {interval_seconds}s interval")
while True:
try:
# Wait for the next scheduled run
await asyncio.sleep(interval_seconds)
# Run demotion job
await self._run_demotion_job()
except asyncio.CancelledError:
logger.info("Demotion scheduler cancelled")
break
except Exception as e:
logger.error(f"Error in demotion scheduler: {e}")
# Continue running despite errors
await asyncio.sleep(60) # Wait 1 minute before retrying
async def _run_demotion_job(self):
"""Execute a single demotion job run"""
job_start = datetime.now(timezone.utc)
logger.info("Starting automatic demotion job")
try:
# Track the job execution with metrics
if metrics_emitter:
async with metrics_emitter.track_operation(
"automatic_demotion_job",
0, # No specific project ID for batch jobs
{"trigger": "scheduled"}
):
# Run the demotion job
stats = await tiering_service.demote_inactive_projects(
batch_size=self.settings.TIERING_BATCH_SIZE
)
# Update task stats
job_end = datetime.now(timezone.utc)
duration = (job_end - job_start).total_seconds()
self.task_stats["demotion_job"].update({
"last_run": job_end.isoformat(),
"last_duration_seconds": duration,
"total_runs": self.task_stats["demotion_job"]["total_runs"] + 1,
"success_count": self.task_stats["demotion_job"]["success_count"] + 1
})
logger.info(f"Demotion job completed successfully: {stats}")
else:
# Fallback without metrics
stats = await tiering_service.demote_inactive_projects(
batch_size=self.settings.TIERING_BATCH_SIZE
)
job_end = datetime.now(timezone.utc)
duration = (job_end - job_start).total_seconds()
self.task_stats["demotion_job"].update({
"last_run": job_end.isoformat(),
"last_duration_seconds": duration,
"total_runs": self.task_stats["demotion_job"]["total_runs"] + 1,
"success_count": self.task_stats["demotion_job"]["success_count"] + 1
})
logger.info(f"Demotion job completed: {stats}")
except Exception as e:
# Update error stats
self.task_stats["demotion_job"]["error_count"] += 1
logger.error(f"Demotion job failed: {e}")
# Emit error metric
if metrics_emitter:
await metrics_emitter._emit_operation_metric(
operation_type="automatic_demotion_job",
status="failure",
duration_ms=0,
project_id=0,
error=str(e)
)
async def _run_metrics_cleanup(self):
"""Clean up old metrics data"""
cleanup_interval = 24 * 3600 # 24 hours
logger.info(f"Starting metrics cleanup task with {cleanup_interval}s interval")
while True:
try:
await asyncio.sleep(cleanup_interval)
logger.info("Running metrics cleanup")
# Clean up old metric events from Redis
from app.core.redis import RedisCache
# Clean events older than 30 days
cutoff_timestamp = int((datetime.now(timezone.utc).timestamp() - (30 * 24 * 3600)))
# Get list of metric keys and clean old ones
pattern = "tiered_storage:events:*"
keys = await RedisCache._redis.keys(pattern)
deleted_count = 0
for key in keys:
key_str = key.decode() if isinstance(key, bytes) else key
timestamp_part = key_str.split(':')[-1]
try:
key_timestamp = int(timestamp_part)
if key_timestamp < cutoff_timestamp:
await RedisCache.delete(key_str)
deleted_count += 1
except (ValueError, IndexError):
# Skip malformed keys
continue
# Trim the operation events list to keep only recent events
list_key = "tiered_storage:operation_events"
await RedisCache.ltrim(list_key, 0, 5000) # Keep last 5k events
logger.info(f"Metrics cleanup completed: deleted {deleted_count} old events")
except asyncio.CancelledError:
logger.info("Metrics cleanup task cancelled")
break
except Exception as e:
logger.error(f"Error in metrics cleanup: {e}")
await asyncio.sleep(3600) # Wait 1 hour before retrying
async def get_task_status(self) -> Dict[str, Any]:
"""Get status of background tasks"""
status = {
"tasks": {},
"statistics": self.task_stats,
"timestamp": datetime.now(timezone.utc).isoformat()
}
for task_name, task in self.running_tasks.items():
status["tasks"][task_name] = {
"running": not task.done(),
"cancelled": task.cancelled() if task.done() else False,
"exception": str(task.exception()) if task.done() and task.exception() else None
}
return status
async def trigger_manual_demotion(self, batch_size: int = None) -> Dict[str, Any]:
"""Manually trigger a demotion job"""
logger.info("Manual demotion job triggered")
try:
stats = await tiering_service.demote_inactive_projects(batch_size)
# Update manual trigger stats
if "manual_triggers" not in self.task_stats:
self.task_stats["manual_triggers"] = {"count": 0, "last_run": None}
self.task_stats["manual_triggers"]["count"] += 1
self.task_stats["manual_triggers"]["last_run"] = datetime.now(timezone.utc).isoformat()
return {
"status": "completed",
"stats": stats,
"timestamp": datetime.now(timezone.utc).isoformat()
}
except Exception as e:
logger.error(f"Manual demotion job failed: {e}")
return {
"status": "failed",
"error": str(e),
"timestamp": datetime.now(timezone.utc).isoformat()
}
# Global instance
background_task_manager = BackgroundTaskManager()
@asynccontextmanager
async def lifespan_manager():
"""Context manager for application lifespan - starts/stops background tasks"""
try:
# Start background tasks
await background_task_manager.start_background_tasks()
yield
finally:
# Stop background tasks
await background_task_manager.stop_background_tasks()