scripts/backend_process_manager.sh

#!/bin/bash

# ATOM Backend Process Manager
# Ensures backend API stays running with auto-recovery and monitoring

set -e

# Configuration
BACKEND_PORT=8001
HEALTH_CHECK_INTERVAL=30
RESTART_DELAY=5
MAX_RESTARTS=10
LOG_FILE="logs/backend_manager.log"
PID_FILE=".backend_manager.pid"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging function
log() {
    local message="$1"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${BLUE}[${timestamp}]${NC} $message" | tee -a "$LOG_FILE"
}

success() {
    log "${GREEN}✅ $1${NC}"
}

warning() {
    log "${YELLOW}⚠️  $1${NC}"
}

error() {
    log "${RED}❌ $1${NC}"
}

# Create necessary directories
setup_directories() {
    mkdir -p logs data tmp
    touch "$LOG_FILE"
    chmod 644 "$LOG_FILE"
}

# Check if backend is healthy
check_backend_health() {
    if curl -s --max-time 5 "http://localhost:$BACKEND_PORT/health" > /dev/null 2>&1; then
        return 0
    else
        return 1
    fi
}

# Get backend process ID
get_backend_pid() {
    lsof -ti:$BACKEND_PORT 2>/dev/null || echo ""
}

# Kill backend processes
kill_backend_processes() {
    local pids=$(get_backend_pid)
    if [ -n "$pids" ]; then
        log "Stopping backend processes: $pids"
        kill $pids 2>/dev/null || true
        sleep 2

        # Force kill if still running
        local remaining_pids=$(get_backend_pid)
        if [ -n "$remaining_pids" ]; then
            log "Force stopping remaining backend processes"
            kill -9 $remaining_pids 2>/dev/null || true
        fi
    fi
}

# Start backend process
start_backend() {
    local attempt=$1
    local backend_pid=""

    log "Starting backend (attempt $attempt)..."

    # Try different backend startup methods in order of preference
    if [ -f "robust_backend.py" ]; then
        python robust_backend.py &
        backend_pid=$!
    elif [ -f "production_backend.py" ]; then
        python production_backend.py &
        backend_pid=$!
    elif [ -f "backend/fixed_main_api_app.py" ]; then
        python backend/fixed_main_api_app.py &
        backend_pid=$!
    elif [ -f "start_simple_backend.py" ]; then
        python start_simple_backend.py &
        backend_pid=$!
    else
        error "No backend startup file found"
        return 1
    fi

    echo $backend_pid > .backend.pid
    log "Backend started with PID: $backend_pid"

    # Wait for backend to be ready
    local health_check_attempts=0
    local max_health_checks=30

    while [ $health_check_attempts -lt $max_health_checks ]; do
        if check_backend_health; then
            success "Backend is healthy and responding on port $BACKEND_PORT"
            return 0
        fi

        # Check if process is still running
        if ! ps -p $backend_pid > /dev/null 2>&1; then
            error "Backend process died during startup"
            return 1
        fi

        sleep 2
        ((health_check_attempts++))
    done

    error "Backend failed to become healthy within timeout"
    kill $backend_pid 2>/dev/null || true
    return 1
}

# Monitor backend health
monitor_backend() {
    local restart_count=0
    local last_restart_time=0
    local consecutive_failures=0

    log "Starting backend monitoring..."

    while true; do
        if ! check_backend_health; then
            ((consecutive_failures++))
            warning "Backend health check failed (consecutive failures: $consecutive_failures)"

            # Check if backend process is still running but unresponsive
            local backend_pid=$(get_backend_pid)
            if [ -n "$backend_pid" ]; then
                warning "Backend process $backend_pid is running but unresponsive - restarting"
            else
                warning "Backend process not found - restarting"
            fi

            # Implement exponential backoff for restarts
            local current_time=$(date +%s)
            local time_since_last_restart=$((current_time - last_restart_time))

            if [ $restart_count -gt 0 ] && [ $time_since_last_restart -lt $((RESTART_DELAY * 2)) ]; then
                local backoff_delay=$((RESTART_DELAY * (2 ** (restart_count - 1))))
                backoff_delay=$((backoff_delay > 300 ? 300 : backoff_delay)) # Cap at 5 minutes
                warning "Implementing exponential backoff: waiting ${backoff_delay}s before restart"
                sleep $backoff_delay
            else
                sleep $RESTART_DELAY
            fi

            # Kill existing backend processes
            kill_backend_processes

            # Start backend
            if start_backend $((restart_count + 1)); then
                ((restart_count++))
                last_restart_time=$(date +%s)
                consecutive_failures=0
                success "Backend restarted successfully (total restarts: $restart_count)"

                # Reset restart count after successful operation for a while
                if [ $restart_count -ge $MAX_RESTARTS ]; then
                    warning "Reached maximum restart count ($MAX_RESTARTS), resetting counter"
                    restart_count=0
                fi
            else
                error "Backend restart failed"
                consecutive_failures=$((consecutive_failures + 1))
            fi
        else
            # Backend is healthy
            if [ $consecutive_failures -gt 0 ]; then
                success "Backend recovered after $consecutive_failures failures"
                consecutive_failures=0
            fi

            # Log periodic health status
            if [ $(( $(date +%s) % 300 )) -eq 0 ]; then # Every 5 minutes
                log "Backend health check passed - running normally"
            fi
        fi

        sleep $HEALTH_CHECK_INTERVAL
    done
}

# Display status information
show_status() {
    echo ""
    echo "🔍 ATOM Backend Process Manager Status"
    echo "======================================"
    echo "Manager PID: $$"
    echo "Backend Port: $BACKEND_PORT"
    echo "Health Check Interval: ${HEALTH_CHECK_INTERVAL}s"
    echo "Log File: $LOG_FILE"
    echo ""

    local backend_pid=$(get_backend_pid)
    if [ -n "$backend_pid" ]; then
        echo "Backend Process:"
        echo "  PID: $backend_pid"
        echo "  Status: $(ps -p $backend_pid -o state= 2>/dev/null || echo "not found")"

        if check_backend_health; then
            echo "  Health: ${GREEN}HEALTHY${NC}"
        else
            echo "  Health: ${RED}UNHEALTHY${NC}"
        fi
    else
        echo "Backend Process: ${RED}NOT RUNNING${NC}"
    fi

    echo ""
    echo "📊 Recent Logs:"
    tail -5 "$LOG_FILE" 2>/dev/null | while read line; do
        echo "  $line"
    done
}

# Cleanup function
cleanup() {
    log "Shutting down backend process manager..."
    kill_backend_processes
    rm -f "$PID_FILE" .backend.pid
    success "Backend process manager shutdown complete"
    exit 0
}

# Main function
main() {
    log "🚀 Starting ATOM Backend Process Manager"

    # Setup
    setup_directories

    # Check if already running
    if [ -f "$PID_FILE" ]; then
        local manager_pid=$(cat "$PID_FILE")
        if ps -p "$manager_pid" > /dev/null 2>&1; then
            error "Backend process manager is already running with PID: $manager_pid"
            exit 1
        else
            warning "Stale PID file found, removing..."
            rm -f "$PID_FILE"
        fi
    fi

    # Create PID file
    echo $$ > "$PID_FILE"

    # Register cleanup handler
    trap cleanup SIGINT SIGTERM

    # Initial backend start
    if ! start_backend 1; then
        error "Initial backend startup failed"
        exit 1
    fi

    # Show initial status
    show_status

    # Start monitoring
    log "Backend process manager is now monitoring the backend service"
    log "Press Ctrl+C to stop monitoring and shutdown all services"

    monitor_backend
}

# Handle command line arguments
case "${1:-}" in
    status)
        show_status
        ;;
    stop)
        if [ -f "$PID_FILE" ]; then
            local manager_pid=$(cat "$PID_FILE")
            kill $manager_pid 2>/dev/null && success "Sent stop signal to process manager" || error "Failed to stop process manager"
        else
            error "Process manager not running"
        fi
        ;;
    restart)
        if [ -f "$PID_FILE" ]; then
            local manager_pid=$(cat "$PID_FILE")
            kill $manager_pid 2>/dev/null && sleep 2
        fi
        exec "$0"
        ;;
    logs)
        tail -f "$LOG_FILE"
        ;;
    health)
        if check_backend_health; then
            echo "HEALTHY"
            exit 0
        else
            echo "UNHEALTHY"
            exit 1
        fi
        ;;
    *)
        main
        ;;
esac