Spaces:

hftestx
/

52c75d7a

Paused

52c75d7a / scripts /utils /restic-backup.sh

Add restic auto-recovery feature and backup service conflict checking

cf5db51 over 1 year ago

10.4 kB

#!/bin/bash

set -e

# Restic Backup Utility Script
# Provides independent backup functionality, can be called by other scripts

# Logging functions
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [BACKUP] $*"
}

log_success() {
    echo -e "\033[32m[$(date '+%Y-%m-%d %H:%M:%S')] [SUCCESS] $*\033[0m"
}

log_error() {
    echo -e "\033[31m[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $*\033[0m"
}

log_warn() {
    echo -e "\033[33m[$(date '+%Y-%m-%d %H:%M:%S')] [WARN] $*\033[0m"
}

# Display usage instructions
show_usage() {
    cat << EOF
Usage: $0 [options] [paths...]
Options:
  -c, --config FILE     Specify configuration file path
  -r, --repository DIR  Specify repository path (overrides config file)
  -p, --password PASS   Specify password (overrides config file)
  -t, --tag TAG         Add backup tag (can be used multiple times)
  -e, --exclude PATTERN Add exclude pattern (can be used multiple times)
  --compression MODE    Compression mode: off, fastest, auto, better, max (default: auto)
  -v, --verbose         Enable verbose output
  -n, --dry-run         Only show operations to be executed
  -h, --help            Show this help message
Compression mode explanation:
  off     - No compression (fastest backup speed)
  fastest - Fastest compression (minimal CPU usage)
  auto    - Auto compression (default, balanced speed and compression ratio)
  better  - Better compression (more CPU, better compression ratio)
  max     - Maximum compression (highest CPU, best compression ratio)
Examples:
  $0                              # Backup using default configuration
  $0 /home/user/data              # Backup specified path
  $0 -t "manual" -v /important    # Verbose backup with tag
  $0 --compression max -v         # Verbose backup with maximum compression
  $0 -c custom.conf -n            # Dry run with custom configuration
EOF
}

# Load configuration file
load_config() {
    local config_file="$1"
    
    if [[ -f "$config_file" ]]; then
        log "Loading configuration file: $config_file"
        
        # Use safe configuration loading method
        local temp_config="/tmp/restic_backup_config_$$"
        sed 's/^\([^#]*\)=\(.*\)$/\1=${\1:-\2}/' "$config_file" > "$temp_config"
        source "$temp_config"
        rm -f "$temp_config"
        
        log_success "Configuration loaded (environment variables take precedence)"
    else
        log_warn "Configuration file does not exist: $config_file"
    fi
}

# Set environment variables
setup_environment() {
    # Check necessary configuration variables
    if [[ -z "$RESTIC_REPOSITORY_PATH" ]]; then
        log_error "RESTIC_REPOSITORY_PATH not set, please check configuration file or environment variables"
        return 1
    fi
    
    # Check password setting
    if [[ -z "${RESTIC_PASSWORD:-}" ]]; then
        log_error "RESTIC_PASSWORD not set, please check configuration file or environment variables"
        return 1
    fi
    
    log "Using password from environment variables"
    
    # Export environment variables
    export RESTIC_REPOSITORY="$RESTIC_REPOSITORY_PATH"
    export RESTIC_PASSWORD
    export RESTIC_CACHE_DIR
    
    # Set performance tuning environment variables (read from config file or environment variables)
    [[ -n "$RESTIC_MAX_PROCS" ]] && export GOMAXPROCS="$RESTIC_MAX_PROCS"
    [[ -n "$RESTIC_TEMP_DIR" ]] && export TMPDIR="$RESTIC_TEMP_DIR"
    [[ -n "$RESTIC_FEATURES" ]] && export RESTIC_FEATURES
    
    # Check restic command
    if ! command -v restic &> /dev/null; then
        log_error "restic command not found"
        return 1
    fi
    
    log_success "Environment setup completed"
}

# Validate backup paths
validate_paths() {
    local paths=("$@")
    local valid_paths=()
    
    for path in "${paths[@]}"; do
        if [[ -e "$path" ]]; then
            valid_paths+=("$path")
            log "Valid path: $path"
        else
            log_warn "Path does not exist: $path"
        fi
    done
    
    if [[ ${#valid_paths[@]} -eq 0 ]]; then
        log_error "No valid backup paths"
        return 1
    fi
    
    # Store valid paths in global variable
    VALIDATED_PATHS=("${valid_paths[@]}")
    return 0
}

# Perform backup
perform_backup() {
    local dry_run="$1"
    shift
    local backup_paths=("$@")
    
    log "Starting backup operation..."
    
    # If no paths specified, use paths from configuration
    if [[ ${#backup_paths[@]} -eq 0 ]]; then
        IFS=':' read -ra backup_paths <<< "$RESTIC_BACKUP_PATHS"
    fi
    
    # Validate paths
    if ! validate_paths "${backup_paths[@]}"; then
        return 1
    fi
    
    # Build backup command arguments
    local args=("backup")
    
    # Add exclude patterns
    if [[ -n "$RESTIC_EXCLUDE_PATTERNS" ]]; then
        IFS=',' read -ra patterns <<< "$RESTIC_EXCLUDE_PATTERNS"
        for pattern in "${patterns[@]}"; do
            args+=("--exclude" "$pattern")
        done
    fi
    
    # Add extra exclude patterns (from command line)
    for exclude in "${EXTRA_EXCLUDES[@]}"; do
        args+=("--exclude" "$exclude")
    done
    
    # Add tags
    if [[ -n "$RESTIC_BACKUP_TAG" ]]; then
        IFS=',' read -ra tags <<< "$RESTIC_BACKUP_TAG"
        for tag in "${tags[@]}"; do
            args+=("--tag" "$tag")
        done
    fi
    
    # Add extra tags (from command line)
    for tag in "${EXTRA_TAGS[@]}"; do
        args+=("--tag" "$tag")
    done
    
    # Add options
    [[ "$RESTIC_VERBOSE" == "true" ]] && args+=("--verbose")
    
    # Add compression option
    args+=("--compression" "$RESTIC_COMPRESSION")
    
    # New: Performance tuning options (prioritize RESTIC_READ_CONCURRENCY, otherwise use RESTIC_PARALLEL_UPLOADS)
    if [[ -n "$RESTIC_READ_CONCURRENCY" ]]; then
        args+=("--read-concurrency" "$RESTIC_READ_CONCURRENCY")
    elif [[ -n "$RESTIC_PARALLEL_UPLOADS" ]]; then
        args+=("--read-concurrency" "$RESTIC_PARALLEL_UPLOADS")
    fi
    [[ -n "$RESTIC_PACK_SIZE" ]] && args+=("--pack-size" "$RESTIC_PACK_SIZE")
    [[ "$RESTIC_NO_SCAN" == "true" ]] && args+=("--no-scan")
    [[ "$RESTIC_EXTRA_VERIFY" == "false" ]] && args+=("--no-extra-verify")
    [[ "$RESTIC_NO_CACHE" == "true" ]] && args+=("--no-cache")
    
    # New: Backend connection configuration
    if [[ -n "$RESTIC_BACKEND_CONNECTIONS" ]]; then
        # Auto-detect backend type
        if [[ "$RESTIC_REPOSITORY_PATH" =~ ^s3: ]]; then
            args+=("-o" "s3.connections=$RESTIC_BACKEND_CONNECTIONS")
        elif [[ "$RESTIC_REPOSITORY_PATH" =~ ^rest: ]]; then
            args+=("-o" "rest.connections=$RESTIC_BACKEND_CONNECTIONS")
        elif [[ "$RESTIC_REPOSITORY_PATH" =~ ^sftp: ]]; then
            args+=("-o" "sftp.connections=$RESTIC_BACKEND_CONNECTIONS")
        else
            # Local backend
            args+=("-o" "local.connections=$RESTIC_BACKEND_CONNECTIONS")
        fi
    fi
    
    # Add paths
    args+=("${VALIDATED_PATHS[@]}")
    
    log "Executing command: restic ${args[*]}"
    
    if [[ "$dry_run" == "true" ]]; then
        log_warn "Dry run mode - actual backup not executed"
        return 0
    fi
    
    # Execute backup
    if restic "${args[@]}"; then
        log_success "Backup completed successfully"
        return 0
    else
        log_error "Backup failed"
        return 1
    fi
}

# Main function
main() {
    # Parse command line arguments
    local config_file=""
    local dry_run="false"
    local backup_paths=()
    local EXTRA_TAGS=()
    local EXTRA_EXCLUDES=()
    
    while [[ $# -gt 0 ]]; do
        case $1 in
            -c|--config)
                config_file="$2"
                shift 2
                ;;
            -r|--repository)
                RESTIC_REPOSITORY_PATH="$2"
                shift 2
                ;;
            -p|--password)
                RESTIC_PASSWORD="$2"
                shift 2
                ;;
            -t|--tag)
                EXTRA_TAGS+=("$2")
                shift 2
                ;;
            -e|--exclude)
                EXTRA_EXCLUDES+=("$2")
                shift 2
                ;;
            --compression)
                RESTIC_COMPRESSION="$2"
                shift 2
                ;;
            -v|--verbose)
                RESTIC_VERBOSE="true"
                shift
                ;;
            -n|--dry-run)
                dry_run="true"
                shift
                ;;
            -h|--help)
                show_usage
                exit 0
                ;;
            -*)
                log_error "Unknown option: $1"
                show_usage
                exit 1
                ;;
            *)
                backup_paths+=("$1")
                shift
                ;;
        esac
    done
    
    log "======= Restic Backup Tool ======="
    
    # Load configuration file
    if [[ -n "$config_file" ]]; then
        load_config "$config_file"
    elif [[ -f "${HOME:-/home/user}/config/restic.conf" ]]; then
        load_config "${HOME:-/home/user}/config/restic.conf"
    fi
    
    # Setup environment
    if ! setup_environment; then
        exit 1
    fi
    
    # Display configuration information
    log "Backup configuration:"
    log "  Repository path: $RESTIC_REPOSITORY_PATH"
    log "  Cache directory: $RESTIC_CACHE_DIR"
    log "  Compression mode: $RESTIC_COMPRESSION"
    log "  Dry run mode: $dry_run"
    log "  Performance tuning:"
    log "    - Backend connections: ${RESTIC_BACKEND_CONNECTIONS:-default}"
    log "    - CPU core limit: ${RESTIC_MAX_PROCS:-all available}"
    log "    - Read concurrency: ${RESTIC_READ_CONCURRENCY:-default}"
    log "    - Pack size: ${RESTIC_PACK_SIZE:-16}MiB"
    log "    - Disable scan: ${RESTIC_NO_SCAN:-false}"
    log "    - Extra verify: ${RESTIC_EXTRA_VERIFY:-true}"
    log "    - Disable cache: ${RESTIC_NO_CACHE:-false}"
    [[ -n "$RESTIC_TEMP_DIR" ]] && log "    - Temp directory: $RESTIC_TEMP_DIR"
    [[ -n "$RESTIC_FEATURES" ]] && log "    - Feature flags: $RESTIC_FEATURES"
    [[ ${#EXTRA_TAGS[@]} -gt 0 ]] && log "  Extra tags: ${EXTRA_TAGS[*]}"
    [[ ${#EXTRA_EXCLUDES[@]} -gt 0 ]] && log "  Extra excludes: ${EXTRA_EXCLUDES[*]}"
    
    # Execute backup
    perform_backup "$dry_run" "${backup_paths[@]}"
}

# If script is executed directly (not sourced)
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
    main "$@"
fi 

	#!/bin/bash

	set -e

	# Restic Backup Utility Script
	# Provides independent backup functionality, can be called by other scripts

	# Logging functions
	log() {
	echo "[$(date '+%Y-%m-%d %H:%M:%S')] [BACKUP] $*"
	}

	log_success() {
	echo -e "\033[32m[$(date '+%Y-%m-%d %H:%M:%S')] [SUCCESS] $*\033[0m"
	}

	log_error() {
	echo -e "\033[31m[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $*\033[0m"
	}

	log_warn() {
	echo -e "\033[33m[$(date '+%Y-%m-%d %H:%M:%S')] [WARN] $*\033[0m"
	}

	# Display usage instructions
	show_usage() {
	cat << EOF
	Usage: $0 [options] [paths...]

	Options:
	-c, --config FILE Specify configuration file path
	-r, --repository DIR Specify repository path (overrides config file)
	-p, --password PASS Specify password (overrides config file)
	-t, --tag TAG Add backup tag (can be used multiple times)
	-e, --exclude PATTERN Add exclude pattern (can be used multiple times)
	--compression MODE Compression mode: off, fastest, auto, better, max (default: auto)
	-v, --verbose Enable verbose output
	-n, --dry-run Only show operations to be executed
	-h, --help Show this help message

	Compression mode explanation:
	off - No compression (fastest backup speed)
	fastest - Fastest compression (minimal CPU usage)
	auto - Auto compression (default, balanced speed and compression ratio)
	better - Better compression (more CPU, better compression ratio)
	max - Maximum compression (highest CPU, best compression ratio)

	Examples:
	$0 # Backup using default configuration
	$0 /home/user/data # Backup specified path
	$0 -t "manual" -v /important # Verbose backup with tag
	$0 --compression max -v # Verbose backup with maximum compression
	$0 -c custom.conf -n # Dry run with custom configuration

	EOF
	}

	# Load configuration file
	load_config() {
	local config_file="$1"

	if [[ -f "$config_file" ]]; then
	log "Loading configuration file: $config_file"

	# Use safe configuration loading method
	local temp_config="/tmp/restic_backup_config_$$"
	sed 's/^\([^#]\)=\(.\)$/\1=${\1:-\2}/' "$config_file" > "$temp_config"
	source "$temp_config"
	rm -f "$temp_config"

	log_success "Configuration loaded (environment variables take precedence)"
	else
	log_warn "Configuration file does not exist: $config_file"
	fi
	}

	# Set environment variables
	setup_environment() {
	# Check necessary configuration variables
	if [[ -z "$RESTIC_REPOSITORY_PATH" ]]; then
	log_error "RESTIC_REPOSITORY_PATH not set, please check configuration file or environment variables"
	return 1
	fi

	# Check password setting
	if [[ -z "${RESTIC_PASSWORD:-}" ]]; then
	log_error "RESTIC_PASSWORD not set, please check configuration file or environment variables"
	return 1
	fi

	log "Using password from environment variables"

	# Export environment variables
	export RESTIC_REPOSITORY="$RESTIC_REPOSITORY_PATH"
	export RESTIC_PASSWORD
	export RESTIC_CACHE_DIR

	# Set performance tuning environment variables (read from config file or environment variables)
	[[ -n "$RESTIC_MAX_PROCS" ]] && export GOMAXPROCS="$RESTIC_MAX_PROCS"
	[[ -n "$RESTIC_TEMP_DIR" ]] && export TMPDIR="$RESTIC_TEMP_DIR"
	[[ -n "$RESTIC_FEATURES" ]] && export RESTIC_FEATURES

	# Check restic command
	if ! command -v restic &> /dev/null; then
	log_error "restic command not found"
	return 1
	fi

	log_success "Environment setup completed"
	}

	# Validate backup paths
	validate_paths() {
	local paths=("$@")
	local valid_paths=()

	for path in "${paths[@]}"; do
	if [[ -e "$path" ]]; then
	valid_paths+=("$path")
	log "Valid path: $path"
	else
	log_warn "Path does not exist: $path"
	fi
	done

	if [[ ${#valid_paths[@]} -eq 0 ]]; then
	log_error "No valid backup paths"
	return 1
	fi

	# Store valid paths in global variable
	VALIDATED_PATHS=("${valid_paths[@]}")
	return 0
	}

	# Perform backup
	perform_backup() {
	local dry_run="$1"
	shift
	local backup_paths=("$@")

	log "Starting backup operation..."

	# If no paths specified, use paths from configuration
	if [[ ${#backup_paths[@]} -eq 0 ]]; then
	IFS=':' read -ra backup_paths <<< "$RESTIC_BACKUP_PATHS"
	fi

	# Validate paths
	if ! validate_paths "${backup_paths[@]}"; then
	return 1
	fi

	# Build backup command arguments
	local args=("backup")

	# Add exclude patterns
	if [[ -n "$RESTIC_EXCLUDE_PATTERNS" ]]; then
	IFS=',' read -ra patterns <<< "$RESTIC_EXCLUDE_PATTERNS"
	for pattern in "${patterns[@]}"; do
	args+=("--exclude" "$pattern")
	done
	fi

	# Add extra exclude patterns (from command line)
	for exclude in "${EXTRA_EXCLUDES[@]}"; do
	args+=("--exclude" "$exclude")
	done

	# Add tags
	if [[ -n "$RESTIC_BACKUP_TAG" ]]; then
	IFS=',' read -ra tags <<< "$RESTIC_BACKUP_TAG"
	for tag in "${tags[@]}"; do
	args+=("--tag" "$tag")
	done
	fi

	# Add extra tags (from command line)
	for tag in "${EXTRA_TAGS[@]}"; do
	args+=("--tag" "$tag")
	done

	# Add options
	[[ "$RESTIC_VERBOSE" == "true" ]] && args+=("--verbose")

	# Add compression option
	args+=("--compression" "$RESTIC_COMPRESSION")

	# New: Performance tuning options (prioritize RESTIC_READ_CONCURRENCY, otherwise use RESTIC_PARALLEL_UPLOADS)
	if [[ -n "$RESTIC_READ_CONCURRENCY" ]]; then
	args+=("--read-concurrency" "$RESTIC_READ_CONCURRENCY")
	elif [[ -n "$RESTIC_PARALLEL_UPLOADS" ]]; then
	args+=("--read-concurrency" "$RESTIC_PARALLEL_UPLOADS")
	fi
	[[ -n "$RESTIC_PACK_SIZE" ]] && args+=("--pack-size" "$RESTIC_PACK_SIZE")
	[[ "$RESTIC_NO_SCAN" == "true" ]] && args+=("--no-scan")
	[[ "$RESTIC_EXTRA_VERIFY" == "false" ]] && args+=("--no-extra-verify")
	[[ "$RESTIC_NO_CACHE" == "true" ]] && args+=("--no-cache")

	# New: Backend connection configuration
	if [[ -n "$RESTIC_BACKEND_CONNECTIONS" ]]; then
	# Auto-detect backend type
	if [[ "$RESTIC_REPOSITORY_PATH" =~ ^s3: ]]; then
	args+=("-o" "s3.connections=$RESTIC_BACKEND_CONNECTIONS")
	elif [[ "$RESTIC_REPOSITORY_PATH" =~ ^rest: ]]; then
	args+=("-o" "rest.connections=$RESTIC_BACKEND_CONNECTIONS")
	elif [[ "$RESTIC_REPOSITORY_PATH" =~ ^sftp: ]]; then
	args+=("-o" "sftp.connections=$RESTIC_BACKEND_CONNECTIONS")
	else
	# Local backend
	args+=("-o" "local.connections=$RESTIC_BACKEND_CONNECTIONS")
	fi
	fi

	# Add paths
	args+=("${VALIDATED_PATHS[@]}")

	log "Executing command: restic ${args[*]}"

	if [[ "$dry_run" == "true" ]]; then
	log_warn "Dry run mode - actual backup not executed"
	return 0
	fi

	# Execute backup
	if restic "${args[@]}"; then
	log_success "Backup completed successfully"
	return 0
	else
	log_error "Backup failed"
	return 1
	fi
	}

	# Main function
	main() {
	# Parse command line arguments
	local config_file=""
	local dry_run="false"
	local backup_paths=()
	local EXTRA_TAGS=()
	local EXTRA_EXCLUDES=()

	while [[ $# -gt 0 ]]; do
	case $1 in
	-c\|--config)
	config_file="$2"
	shift 2
	;;
	-r\|--repository)
	RESTIC_REPOSITORY_PATH="$2"
	shift 2
	;;
	-p\|--password)
	RESTIC_PASSWORD="$2"
	shift 2
	;;
	-t\|--tag)
	EXTRA_TAGS+=("$2")
	shift 2
	;;
	-e\|--exclude)
	EXTRA_EXCLUDES+=("$2")
	shift 2
	;;
	--compression)
	RESTIC_COMPRESSION="$2"
	shift 2
	;;
	-v\|--verbose)
	RESTIC_VERBOSE="true"
	shift
	;;
	-n\|--dry-run)
	dry_run="true"
	shift
	;;
	-h\|--help)
	show_usage
	exit 0
	;;
	-*)
	log_error "Unknown option: $1"
	show_usage
	exit 1
	;;
	*)
	backup_paths+=("$1")
	shift
	;;
	esac
	done

	log "======= Restic Backup Tool ======="

	# Load configuration file
	if [[ -n "$config_file" ]]; then
	load_config "$config_file"
	elif [[ -f "${HOME:-/home/user}/config/restic.conf" ]]; then
	load_config "${HOME:-/home/user}/config/restic.conf"
	fi

	# Setup environment
	if ! setup_environment; then
	exit 1
	fi

	# Display configuration information
	log "Backup configuration:"
	log " Repository path: $RESTIC_REPOSITORY_PATH"
	log " Cache directory: $RESTIC_CACHE_DIR"
	log " Compression mode: $RESTIC_COMPRESSION"
	log " Dry run mode: $dry_run"
	log " Performance tuning:"
	log " - Backend connections: ${RESTIC_BACKEND_CONNECTIONS:-default}"
	log " - CPU core limit: ${RESTIC_MAX_PROCS:-all available}"
	log " - Read concurrency: ${RESTIC_READ_CONCURRENCY:-default}"
	log " - Pack size: ${RESTIC_PACK_SIZE:-16}MiB"
	log " - Disable scan: ${RESTIC_NO_SCAN:-false}"
	log " - Extra verify: ${RESTIC_EXTRA_VERIFY:-true}"
	log " - Disable cache: ${RESTIC_NO_CACHE:-false}"
	[[ -n "$RESTIC_TEMP_DIR" ]] && log " - Temp directory: $RESTIC_TEMP_DIR"
	[[ -n "$RESTIC_FEATURES" ]] && log " - Feature flags: $RESTIC_FEATURES"
	[[ ${#EXTRA_TAGS[@]} -gt 0 ]] && log " Extra tags: ${EXTRA_TAGS[*]}"
	[[ ${#EXTRA_EXCLUDES[@]} -gt 0 ]] && log " Extra excludes: ${EXTRA_EXCLUDES[*]}"

	# Execute backup
	perform_backup "$dry_run" "${backup_paths[@]}"
	}

	# If script is executed directly (not sourced)
	if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
	main "$@"
	fi