Buckets:

download
raw
15.8 kB
"""
Scheduler Service - Background job scheduling for transformation models
Uses APScheduler for cron-based scheduling with database persistence
"""
import logging
from datetime import datetime
from typing import Optional, List, Dict, Any
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.jobstores.base import JobLookupError
import pytz
from connection_manager import connection_manager
from transformations.model_loader import ModelLoader
from orchestration import TransformationEngine
from pathlib import Path
# Initialize model loader
models_dir = Path(__file__).parent / "models"
model_loader = ModelLoader(models_dir=str(models_dir))
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class SchedulerService:
"""Manages scheduled execution of transformation models"""
def __init__(self):
self.scheduler = BackgroundScheduler(
timezone=pytz.UTC,
job_defaults={
'coalesce': True, # Combine missed executions
'max_instances': 1 # One instance per job
}
)
self.scheduler.start()
logger.info("Scheduler service started")
def shutdown(self):
"""Gracefully shutdown the scheduler"""
self.scheduler.shutdown(wait=True)
logger.info("Scheduler service stopped")
def add_schedule(
self,
schedule_id: int,
model_names: list,
cron_expression: str,
timezone: str = 'UTC',
model_name: str = None # Backward compatibility
) -> bool:
"""
Add a scheduled job for one or more models
Args:
schedule_id: Database ID of the schedule
model_names: List of model names to run
cron_expression: Cron expression (e.g., "0 9 * * *" for 9am daily)
timezone: Timezone for the schedule
model_name: Single model name (for backward compatibility)
Returns:
True if successfully added
"""
try:
# Handle backward compatibility
if model_name and not model_names:
model_names = [model_name]
if not model_names:
raise ValueError("No models specified for schedule")
# Parse cron expression
parts = cron_expression.split()
if len(parts) != 5:
raise ValueError(f"Invalid cron expression: {cron_expression}")
minute, hour, day, month, day_of_week = parts
# Create cron trigger
trigger = CronTrigger(
minute=minute,
hour=hour,
day=day,
month=month,
day_of_week=day_of_week,
timezone=pytz.timezone(timezone)
)
# Add job to scheduler
job_id = f"schedule_{schedule_id}"
models_desc = ', '.join(model_names) if len(model_names) <= 3 else f"{len(model_names)} models"
self.scheduler.add_job(
func=self._execute_scheduled_models,
trigger=trigger,
id=job_id,
args=[schedule_id, model_names],
replace_existing=True,
name=f"Models: {models_desc}"
)
# Update next_run_at in database
next_run = self.scheduler.get_job(job_id).next_run_time
self._update_next_run_time(schedule_id, next_run)
logger.info(f"Added schedule {schedule_id} for {len(model_names)} model(s) with cron '{cron_expression}'")
return True
except Exception as e:
logger.error(f"Failed to add schedule {schedule_id}: {e}")
return False
def remove_schedule(self, schedule_id: int) -> bool:
"""Remove a scheduled job"""
try:
job_id = f"schedule_{schedule_id}"
self.scheduler.remove_job(job_id)
logger.info(f"Removed schedule {schedule_id}")
return True
except JobLookupError:
logger.warning(f"Schedule {schedule_id} not found in scheduler")
return False
except Exception as e:
logger.error(f"Failed to remove schedule {schedule_id}: {e}")
return False
def pause_schedule(self, schedule_id: int) -> bool:
"""Pause a scheduled job"""
try:
job_id = f"schedule_{schedule_id}"
self.scheduler.pause_job(job_id)
logger.info(f"Paused schedule {schedule_id}")
return True
except Exception as e:
logger.error(f"Failed to pause schedule {schedule_id}: {e}")
return False
def resume_schedule(self, schedule_id: int) -> bool:
"""Resume a paused scheduled job"""
try:
job_id = f"schedule_{schedule_id}"
self.scheduler.resume_job(job_id)
logger.info(f"Resumed schedule {schedule_id}")
return True
except Exception as e:
logger.error(f"Failed to resume schedule {schedule_id}: {e}")
return False
def get_active_schedules(self) -> List[Dict[str, Any]]:
"""Get all currently scheduled jobs"""
jobs = []
for job in self.scheduler.get_jobs():
jobs.append({
'id': job.id,
'name': job.name,
'next_run_time': job.next_run_time.isoformat() if job.next_run_time else None,
'trigger': str(job.trigger)
})
return jobs
def _execute_scheduled_models(self, schedule_id: int, model_names: List[str]):
"""
Execute a scheduled run for one or more models
Args:
schedule_id: Database ID of the schedule
model_names: List of model names to execute
"""
started_at = datetime.now()
run_records = {} # Track run_id for each model
try:
models_str = ', '.join(model_names)
logger.info(f"Starting scheduled run for models '{models_str}' (schedule {schedule_id})")
# Load all models (need dependencies)
all_models = model_loader.load_all_models()
# Find all target models
target_models = []
for model_name in model_names:
target_model = next((m for m in all_models if m.name == model_name), None)
if not target_model:
logger.error(f"Model '{model_name}' not found")
continue
target_models.append(target_model)
if not target_models:
raise ValueError(f"No valid models found from: {model_names}")
# Get all dependencies for all target models
all_dependencies = []
for target_model in target_models:
dependency_models = self._get_dependencies(target_model.name, all_models)
for dep in dependency_models:
if dep not in all_dependencies:
all_dependencies.append(dep)
# Combine dependencies and target models
models_to_run = all_dependencies + target_models
# Create run records for each target model
for target_model in target_models:
run_id = self._create_run_record(schedule_id, target_model.name, started_at)
run_records[target_model.name] = run_id
# Run all models together
engine = TransformationEngine(models_to_run)
context = engine.run(verbose=False)
# Get summary
summary = context.get_summary()
completed_at = datetime.now()
execution_time = (completed_at - started_at).total_seconds()
# Update run records for each target model
for target_model in target_models:
run_id = run_records.get(target_model.name)
if not run_id:
continue
if target_model.status == "completed":
self._complete_run_record(
run_id=run_id,
status='completed',
completed_at=completed_at,
execution_time=execution_time,
dependencies_run=len(all_dependencies),
models_completed=summary['successes'],
models_failed=summary['failures'],
models_skipped=0
)
logger.info(f"Scheduled run completed successfully for '{target_model.name}'")
else:
error_msg = target_model.error or 'Model execution failed'
self._complete_run_record(
run_id=run_id,
status='failed',
completed_at=completed_at,
execution_time=execution_time,
error_message=error_msg,
error_traceback=None
)
logger.error(f"Scheduled run failed for '{target_model.name}': {error_msg}")
except Exception as e:
logger.error(f"Scheduled run failed: {e}")
completed_at = datetime.now()
execution_time = (completed_at - started_at).total_seconds()
# Mark all run records as failed
for model_name, run_id in run_records.items():
self._complete_run_record(
run_id=run_id,
status='failed',
completed_at=completed_at,
execution_time=execution_time,
error_message=str(e),
error_traceback=None
)
finally:
# Update last_run_at and next_run_at for the schedule
self._update_last_run_time(schedule_id, started_at)
job_id = f"schedule_{schedule_id}"
try:
next_run = self.scheduler.get_job(job_id).next_run_time
self._update_next_run_time(schedule_id, next_run)
except:
pass
def _get_dependencies(self, model_name: str, all_models: List) -> List:
"""Recursively get all dependencies for a model"""
model = next((m for m in all_models if m.name == model_name), None)
if not model:
return []
deps = []
for dep_name in model.depends_on:
deps.extend(self._get_dependencies(dep_name, all_models))
dep_model = next((m for m in all_models if m.name == dep_name), None)
if dep_model and dep_model not in deps:
deps.append(dep_model)
return deps
def _create_run_record(self, schedule_id: int, model_name: str, started_at: datetime) -> int:
"""Create a run record in the database"""
with connection_manager.get_connection() as pg:
result = pg.execute("""
INSERT INTO schedule_runs (
schedule_id, model_name, status, started_at
)
VALUES (%s, %s, %s, %s)
RETURNING id
""", params=(schedule_id, model_name, 'running', started_at), fetch=True)
return result[0]['id']
def _complete_run_record(
self,
run_id: int,
status: str,
completed_at: datetime,
execution_time: float,
dependencies_run: int = 0,
models_completed: int = 0,
models_failed: int = 0,
models_skipped: int = 0,
error_message: Optional[str] = None,
error_traceback: Optional[str] = None
):
"""Update a run record with completion details"""
with connection_manager.get_connection() as pg:
pg.execute("""
UPDATE schedule_runs
SET
status = %s,
completed_at = %s,
execution_time_seconds = %s,
dependencies_run = %s,
models_completed = %s,
models_failed = %s,
models_skipped = %s,
error_message = %s,
error_traceback = %s
WHERE id = %s
""", params=(
status, completed_at, execution_time,
dependencies_run, models_completed, models_failed, models_skipped,
error_message, error_traceback,
run_id
))
def _update_last_run_time(self, schedule_id: int, last_run_at: datetime):
"""Update the last_run_at timestamp for a schedule"""
with connection_manager.get_connection() as pg:
pg.execute("""
UPDATE model_schedules
SET last_run_at = %s, updated_at = CURRENT_TIMESTAMP
WHERE id = %s
""", params=(last_run_at, schedule_id))
def _update_next_run_time(self, schedule_id: int, next_run_at: datetime):
"""Update the next_run_at timestamp for a schedule"""
with connection_manager.get_connection() as pg:
pg.execute("""
UPDATE model_schedules
SET next_run_at = %s, updated_at = CURRENT_TIMESTAMP
WHERE id = %s
""", params=(next_run_at, schedule_id))
def load_schedules_from_db(self):
"""Load all active schedules from database and add them to scheduler"""
try:
with connection_manager.get_connection() as pg:
# Get all active schedules
schedules = pg.execute("""
SELECT id, schedule_name, cron_expression, timezone, is_active
FROM model_schedules
WHERE is_active = TRUE
ORDER BY id
""", fetch=True)
# For each schedule, get its associated models
for schedule in schedules:
models = pg.execute("""
SELECT model_name
FROM schedule_models
WHERE schedule_id = %s
ORDER BY model_name
""", params=(schedule['id'],), fetch=True)
model_names = [m['model_name'] for m in models]
if model_names:
self.add_schedule(
schedule_id=schedule['id'],
model_names=model_names,
cron_expression=schedule['cron_expression'],
timezone=schedule['timezone']
)
else:
logger.warning(f"Schedule {schedule['id']} has no models assigned, skipping")
logger.info(f"Loaded {len(schedules)} active schedules from database")
except Exception as e:
logger.error(f"Failed to load schedules from database: {e}")
# Global scheduler instance
_scheduler_service: Optional[SchedulerService] = None
def get_scheduler() -> SchedulerService:
"""Get or create the global scheduler service instance"""
global _scheduler_service
if _scheduler_service is None:
_scheduler_service = SchedulerService()
return _scheduler_service
def start_scheduler():
"""Initialize and start the scheduler service"""
scheduler = get_scheduler()
scheduler.load_schedules_from_db()
return scheduler
def stop_scheduler():
"""Stop the scheduler service"""
global _scheduler_service
if _scheduler_service:
_scheduler_service.shutdown()
_scheduler_service = None

Xet Storage Details

Size:
15.8 kB
·
Xet hash:
8685cb5307660d31db3dcbf81a4a0cf660fd39c2d761e1ca8713da001f77ef95

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.