📦 Backup and Rotation Recipes

Effective backup and rotation strategies ensure data protection while managing storage efficiently. This recipe provides proven patterns for implementing reliable backup systems with proper retention policies.

---

🎯 Core Principles

Incremental Backup Strategy

Combine full and incremental backups for efficient storage usage.

#!/bin/bash
# backup-strategy.sh - Comprehensive backup system

# Configuration
BACKUP_ROOT="/backup"
FULL_BACKUP_DAY="Sunday"  # Weekly full backup
RETENTION_DAYS=30
RETENTION_WEEKS=12
RETENTION_MONTHS=12

# Timestamp functions
get_timestamp() {
    date +%Y%m%d_%H%M%S
}

get_date_only() {
    date +%Y%m%d
}

get_day_of_week() {
    date +%A
}

# Backup type determination
should_do_full_backup() {
    [ "$(get_day_of_week)" = "$FULL_BACKUP_DAY" ]
}

# Directory structure
create_backup_dirs() {
    local backup_base="$1"

    mkdir -p "$backup_base"/{daily,weekly,monthly,incoming}
    mkdir -p "$backup_base"/logs

    # Set proper permissions
    chmod 750 "$backup_base"
    chown root:backup "$backup_base"
}

# Full backup implementation
perform_full_backup() {
    local source_dir="$1"
    local backup_base="$2"
    local backup_name="$3"

    local timestamp=$(get_timestamp)
    local backup_path="$backup_base/incoming/${backup_name}_full_${timestamp}.tar.gz"
    local log_file="$backup_base/logs/${backup_name}_full_${timestamp}.log"

    echo "Starting full backup of $source_dir" | tee "$log_file"

    # Create full backup
    if tar -czf "$backup_path" -C "$(dirname "$source_dir")" "$(basename "$source_dir")" 2>>"$log_file"; then
        echo "Full backup completed: $backup_path" | tee -a "$log_file"

        # Move to daily directory
        mv "$backup_path" "$backup_base/daily/"

        # Create symlink for latest
        ln -sf "$(basename "$backup_path")" "$backup_base/latest_full.tar.gz"

        return 0
    else
        echo "Full backup failed" | tee -a "$log_file"
        return 1
    fi
}

# Incremental backup implementation
perform_incremental_backup() {
    local source_dir="$1"
    local backup_base="$2"
    local backup_name="$3"

    local timestamp=$(get_timestamp)
    local backup_path="$backup_base/incoming/${backup_name}_incr_${timestamp}.tar.gz"
    local log_file="$backup_base/logs/${backup_name}_incr_${timestamp}.log"

    # Find last backup for comparison
    local last_backup
    last_backup=$(find_last_backup "$backup_base")

    if [ -z "$last_backup" ]; then
        echo "No previous backup found, performing full backup instead" | tee "$log_file"
        perform_full_backup "$source_dir" "$backup_base" "$backup_name"
        return $?
    fi

    echo "Starting incremental backup of $source_dir since $last_backup" | tee -a "$log_file"

    # Create incremental backup using tar with snapshot file
    local snapshot_file="$backup_base/incoming/${backup_name}_snapshot_${timestamp}.snar"

    if tar -czf "$backup_path" -g "$snapshot_file" -C "$(dirname "$source_dir")" "$(basename "$source_dir")" 2>>"$log_file"; then
        echo "Incremental backup completed: $backup_path" | tee -a "$log_file"

        # Move files to appropriate directories
        mv "$backup_path" "$backup_base/daily/"
        mv "$snapshot_file" "$backup_base/daily/"

        # Create symlink for latest
        ln -sf "$(basename "$backup_path")" "$backup_base/latest_incr.tar.gz"

        return 0
    else
        echo "Incremental backup failed" | tee -a "$log_file"
        rm -f "$snapshot_file"  # Cleanup failed snapshot
        return 1
    fi
}

# Find last backup for incremental comparison
find_last_backup() {
    local backup_base="$1"

    # Look for most recent snapshot file
    find "$backup_base/daily" -name "*.snar" -type f | sort | tail -1
}

---

🔧 Database Backup Patterns

MySQL Backup with Point-in-Time Recovery

# mysql-backup.sh - MySQL backup with binary log support

MYSQL_BACKUP_CONFIG="/etc/mysql/backup.conf"
source "$MYSQL_BACKUP_CONFIG"

backup_mysql_database() {
    local database_name="$1"
    local backup_dir="$2"
    local timestamp=$(date +%Y%m%d_%H%M%S)

    local backup_file="$backup_dir/${database_name}_${timestamp}.sql"
    local binlog_position_file="$backup_dir/${database_name}_${timestamp}_binlog.pos"

    # Create backup directory
    mkdir -p "$backup_dir"

    # Get master status for point-in-time recovery
    local master_status
    master_status=$(mysql -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "SHOW MASTER STATUS\G" 2>/dev/null)

    # Extract binary log position
    local binlog_file
    local binlog_position
    binlog_file=$(echo "$master_status" | grep "File:" | awk '{print $2}')
    binlog_position=$(echo "$master_status" | grep "Position:" | awk '{print $2}')

    # Save binary log position
    echo "Binary Log File: $binlog_file" > "$binlog_position_file"
    echo "Binary Log Position: $binlog_position" >> "$binlog_position_file"

    # Perform database dump
    if mysqldump -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" \
        --single-transaction \
        --routines \
        --triggers \
        --events \
        "$database_name" > "$backup_file"; then

        # Compress backup
        gzip "$backup_file"

        echo "MySQL backup completed: ${backup_file}.gz"
        echo "Binary log position saved: $binlog_position_file"
        return 0
    else
        echo "MySQL backup failed" >&2
        rm -f "$backup_file" "$binlog_position_file"
        return 1
    fi
}

# Point-in-time recovery function
recover_mysql_point_in_time() {
    local backup_file="$1"
    local target_datetime="$2"
    local database_name="$3"

    # Restore from backup
    gunzip -c "$backup_file" | mysql -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$database_name"

    # Apply binary logs from backup position to target time
    # This would require additional logic to parse binary logs
    echo "Point-in-time recovery initiated to $target_datetime"
}

---

🗃️ File System Backup

Rsync-Based Incremental Backup

# rsync-backup.sh - Efficient incremental file backup

perform_rsync_backup() {
    local source="$1"
    local destination="$2"
    local backup_name="$3"

    local timestamp=$(get_timestamp)
    local backup_path="$destination/$backup_name/$timestamp"
    local latest_link="$destination/$backup_name/latest"
    local log_file="$destination/logs/${backup_name}_${timestamp}.log"

    # Create backup directory
    mkdir -p "$backup_path"
    mkdir -p "$(dirname "$log_file")"

    echo "Starting rsync backup from $source to $backup_path" | tee "$log_file"

    # Perform rsync with hard links for efficiency
    if rsync -avH --delete \
        --link-dest="$latest_link" \
        "$source/" "$backup_path/" 2>>"$log_file"; then

        echo "Rsync backup completed successfully" | tee -a "$log_file"

        # Update latest symlink
        rm -f "$latest_link"
        ln -s "$timestamp" "$latest_link"

        return 0
    else
        echo "Rsync backup failed" | tee -a "$log_file"
        rm -rf "$backup_path"  # Cleanup failed backup
        return 1
    fi
}

# Hard link-based rotation (rsnapshot style)
create_hardlink_rotation() {
    local backup_root="$1"
    local backup_name="$2"
    local max_backups="$3"

    # Rotate existing backups
    for ((i=max_backups; i>=1; i--)); do
        local current="$backup_root/$backup_name.$i"
        local next="$backup_root/$backup_name.$((i+1))"

        if [ -d "$current" ]; then
            if [ $i -eq $max_backups ]; then
                # Remove oldest backup
                rm -rf "$current"
            else
                # Rotate backup
                mv "$current" "$next"
            fi
        fi
    done

    # Create new backup slot
    mkdir -p "$backup_root/$backup_name.1"
}

---

🔄 Rotation Policies

Time-Based Retention Management

# retention-manager.sh - Automated backup retention

enforce_retention_policy() {
    local backup_base="$1"
    local retention_days="$2"
    local retention_weeks="$3"
    local retention_months="$4"

    local current_date
    current_date=$(date +%s)

    echo "Enforcing retention policy on $backup_base"

    # Daily retention (keep last N days)
    enforce_daily_retention "$backup_base/daily" "$retention_days"

    # Weekly retention (keep one per week)
    enforce_weekly_retention "$backup_base/daily" "$retention_weeks"

    # Monthly retention (keep one per month)
    enforce_monthly_retention "$backup_base/daily" "$retention_months"
}

enforce_daily_retention() {
    local backup_dir="$1"
    local keep_days="$2"

    local cutoff_date
    cutoff_date=$(date -d "$keep_days days ago" +%Y%m%d)

    echo "Removing daily backups older than $cutoff_date"

    find "$backup_dir" -name "*.tar.gz" -type f | while read -r backup_file; do
        local file_date
        file_date=$(basename "$backup_file" | cut -d_ -f1)

        if [ "$file_date" -lt "$cutoff_date" ]; then
            echo "Removing old backup: $backup_file"
            rm -f "$backup_file"

            # Remove associated snapshot files
            local snapshot_file
            snapshot_file="${backup_file%.tar.gz}.snar"
            if [ -f "$snapshot_file" ]; then
                rm -f "$snapshot_file"
            fi
        fi
    done
}

enforce_weekly_retention() {
    local backup_dir="$1"
    local keep_weeks="$2"

    echo "Keeping one backup per week for $keep_weeks weeks"

    # Group backups by week and keep newest from each week
    # Implementation would involve date manipulation and grouping logic
}

enforce_monthly_retention() {
    local backup_dir="$1"
    local keep_months="$2"

    echo "Keeping one backup per month for $keep_months months"

    # Similar grouping logic for monthly retention
}

---

🛡️ Backup Verification

Automated Backup Integrity Checking

# backup-verifier.sh - Backup integrity validation

verify_backup_integrity() {
    local backup_file="$1"
    local backup_type="$2"  # tar, sql, etc.

    case "$backup_type" in
        tar)
            verify_tar_backup "$backup_file"
            ;;
        sql)
            verify_sql_backup "$backup_file"
            ;;
        rsync)
            verify_rsync_backup "$backup_file"
            ;;
        *)
            echo "Unknown backup type: $backup_type" >&2
            return 1
            ;;
    esac
}

verify_tar_backup() {
    local tar_file="$1"

    echo "Verifying tar backup: $tar_file"

    # Check if file exists
    if [ ! -f "$tar_file" ]; then
        echo "Error: Backup file not found: $tar_file" >&2
        return 1
    fi

    # Verify tar file integrity
    if tar -tzf "$tar_file" >/dev/null 2>&1; then
        echo "Tar backup integrity check passed"
        return 0
    else
        echo "Error: Tar backup integrity check failed" >&2
        return 1
    fi
}

verify_sql_backup() {
    local sql_file="$1"

    echo "Verifying SQL backup: $sql_file"

    # Check file existence
    if [ ! -f "$sql_file" ]; then
        echo "Error: SQL backup file not found: $sql_file" >&2
        return 1
    fi

    # Basic syntax check (MySQL example)
    if gunzip -c "$sql_file" | head -100 | grep -E "^(CREATE|INSERT|USE)" >/dev/null; then
        echo "SQL backup syntax check passed"
        return 0
    else
        echo "Error: SQL backup syntax check failed" >&2
        return 1
    fi
}

# Automated verification scheduler
schedule_backup_verification() {
    local backup_dir="$1"
    local interval_hours="${2:-24}"  # Default 24 hours

    # Find recent backups to verify
    find "$backup_dir" -name "*.tar.gz" -type f -mtime -1 | while read -r backup_file; do
        if ! verify_backup_integrity "$backup_file" "tar"; then
            echo "Backup verification failed: $backup_file" >&2
            # Send alert or notification
        fi
    done
}

---

🎨 Advanced Backup Features

Encrypted Backup Storage

# encrypted-backup.sh - Backup with encryption

perform_encrypted_backup() {
    local source_dir="$1"
    local backup_base="$2"
    local backup_name="$3"
    local encryption_key_file="$4"

    local timestamp=$(get_timestamp)
    local plain_backup="$backup_base/incoming/${backup_name}_${timestamp}.tar"
    local encrypted_backup="$backup_base/daily/${backup_name}_${timestamp}.tar.gpg"
    local log_file="$backup_base/logs/${backup_name}_${timestamp}.log"

    # Create plain backup
    if tar -cf "$plain_backup" -C "$(dirname "$source_dir")" "$(basename "$source_dir")"; then
        echo "Plain backup created: $plain_backup" | tee "$log_file"

        # Encrypt backup
        if gpg --batch --yes --passphrase-file "$encryption_key_file" \
            --symmetric --cipher-algo AES256 \
            --output "$encrypted_backup" "$plain_backup"; then

            echo "Encrypted backup created: $encrypted_backup" | tee -a "$log_file"

            # Remove plain backup
            rm -f "$plain_backup"

            return 0
        else
            echo "Encryption failed" | tee -a "$log_file"
            rm -f "$plain_backup"
            return 1
        fi
    else
        echo "Plain backup failed" | tee "$log_file"
        return 1
    fi
}

# Decryption function for restores
restore_encrypted_backup() {
    local encrypted_file="$1"
    local destination="$2"
    local decryption_key_file="$3"

    local temp_file
    temp_file=$(mktemp)

    # Decrypt backup
    if gpg --batch --yes --passphrase-file "$decryption_key_file" \
        --decrypt --output "$temp_file" "$encrypted_file"; then

        # Extract backup
        if tar -xf "$temp_file" -C "$destination"; then
            echo "Backup restored successfully to $destination"
            rm -f "$temp_file"
            return 0
        else
            echo "Backup extraction failed" >&2
            rm -f "$temp_file"
            return 1
        fi
    else
        echo "Decryption failed" >&2
        rm -f "$temp_file"
        return 1
    fi
}

---

🧾 Summary Best Practices

Backup Strategy Guidelines

1. Regular Schedule: Implement consistent backup schedules
2. Multiple Copies: Follow 3-2-1 rule (3 copies, 2 media, 1 offsite)
3. Verification: Regularly test backup integrity
4. Retention: Implement appropriate retention policies
5. Security: Encrypt sensitive backups
6. Monitoring: Monitor backup success/failure
7. Documentation: Maintain clear backup procedures

Key Configuration Template

# backup-config-example.sh - Backup configuration template

# Global settings
export BACKUP_ROOT="/backup"
export LOG_DIR="/var/log/backups"
export RETENTION_DAYS=30
export RETENTION_WEEKS=12
export RETENTION_MONTHS=12

# Database settings
export MYSQL_USER="backup_user"
export MYSQL_PASSWORD_FILE="/etc/mysql/backup.password"
export POSTGRES_USER="backup_user"
export POSTGRES_PASSWORD_FILE="/etc/postgresql/backup.password"

# Encryption settings
export ENCRYPTION_KEY_FILE="/etc/backup/encryption.key"
export GPG_RECIPIENT="backup-admin@example.com"

# Notification settings
export ALERT_EMAIL="admin@example.com"
export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK"

# Storage settings
export S3_BUCKET="company-backups"
export S3_REGION="us-west-2"

---

🧠 Complete Backup Script Example

#!/bin/bash
# complete-backup-system.sh - Production-ready backup solution

set -euo pipefail

# Load configuration
CONFIG_FILE="${1:-/etc/backup/config}"
if [ -f "$CONFIG_FILE" ]; then
    source "$CONFIG_FILE"
fi

# Default configuration
: ${BACKUP_ROOT:="/backup"}
: ${SOURCE_DIR:="/data"}
: ${BACKUP_NAME:="data_backup"}
: ${FULL_BACKUP_DAY:="Sunday"}
: ${RETENTION_DAYS:=30}

# Main backup function
main() {
    local timestamp
    timestamp=$(date +%Y%m%d_%H%M%S)

    echo "[$timestamp] Starting backup process"

    # Create directory structure
    create_backup_dirs "$BACKUP_ROOT"

    # Determine backup type
    if should_do_full_backup; then
        echo "Performing full backup"
        perform_full_backup "$SOURCE_DIR" "$BACKUP_ROOT" "$BACKUP_NAME"
    else
        echo "Performing incremental backup"
        perform_incremental_backup "$SOURCE_DIR" "$BACKUP_ROOT" "$BACKUP_NAME"
    fi

    # Enforce retention policy
    enforce_retention_policy "$BACKUP_ROOT" "$RETENTION_DAYS" 4 12

    # Verify recent backups
    schedule_backup_verification "$BACKUP_ROOT"

    echo "[$timestamp] Backup process completed"
}

# Error handling
trap 'echo "Backup failed at line $LINENO"; exit 1' ERR

# Run main function
main "$@"

---

🧾 See Also

• Recipes: Healthchecks and Probes
• Recipes: Log Processing
• Patterns: Idempotent Operations
• Linux Namespaces and CGroups
• Anti-Patterns: Silent Failures
• Disaster Recovery and Backup Best Practices

---