Przejdź do treści

🚀 Migration and Batch Jobs Recipes

Migration and batch processing require careful planning, robust error handling, and comprehensive monitoring. This recipe provides patterns for executing reliable data migrations, batch operations, and large-scale processing jobs.

🎯 Core Principles

Phased Migration Approach

Break complex migrations into manageable, reversible phases.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
#!/bin/bash
# migration-framework.sh - Structured migration framework

# Migration job configuration
MIGRATION_CONFIG_DIR="/etc/migrations"
MIGRATION_LOG_DIR="/var/log/migrations"
MIGRATION_STATE_DIR="/var/lib/migrations"
MIGRATION_LOCK_FILE="/var/run/migration.lock"

# Migration states
STATE_PENDING="pending"
STATE_RUNNING="running"
STATE_COMPLETED="completed"
STATE_FAILED="failed"
STATE_ROLLED_BACK="rolled_back"

# Initialize migration environment
init_migration() {
    local migration_name="$1"

    # Create directories
    mkdir -p "$MIGRATION_LOG_DIR"
    mkdir -p "$MIGRATION_STATE_DIR"

    # Set up logging
    exec 3>&1 4>&2
    exec 1>>"$MIGRATION_LOG_DIR/${migration_name}.log"
    exec 2>&1

    echo "[$(date)] Initializing migration: $migration_name"
}

# Migration state management
get_migration_state() {
    local migration_name="$1"
    local state_file="$MIGRATION_STATE_DIR/${migration_name}.state"

    if [ -f "$state_file" ]; then
        cat "$state_file"
    else
        echo "$STATE_PENDING"
    fi
}

set_migration_state() {
    local migration_name="$1"
    local state="$2"
    local state_file="$MIGRATION_STATE_DIR/${migration_name}.state"

    echo "$state" > "$state_file"
    echo "[$(date)] Migration $migration_name state set to: $state"
}

# Lock management for concurrent execution prevention
acquire_migration_lock() {
    local migration_name="$1"

    if [ -f "$MIGRATION_LOCK_FILE" ]; then
        local existing_lock
        existing_lock=$(cat "$MIGRATION_LOCK_FILE")
        echo "[$(date)] Migration lock held by: $existing_lock" >&2
        return 1
    fi

    echo "$migration_name" > "$MIGRATION_LOCK_FILE"
    echo "[$(date)] Acquired migration lock for: $migration_name"
    return 0
}

release_migration_lock() {
    local migration_name="$1"

    if [ -f "$MIGRATION_LOCK_FILE" ]; then
        local lock_holder
        lock_holder=$(cat "$MIGRATION_LOCK_FILE")

        if [ "$lock_holder" = "$migration_name" ]; then
            rm -f "$MIGRATION_LOCK_FILE"
            echo "[$(date)] Released migration lock for: $migration_name"
        else
            echo "[$(date)] Warning: Attempted to release lock held by $lock_holder" >&2
        fi
    fi
}

# Migration checkpointing for resume capability
create_checkpoint() {
    local migration_name="$1"
    local checkpoint_name="$2"
    local checkpoint_data="$3"
    local checkpoint_file="$MIGRATION_STATE_DIR/${migration_name}_${checkpoint_name}.ckpt"

    echo "$checkpoint_data" > "$checkpoint_file"
    echo "[$(date)] Created checkpoint: $checkpoint_name for migration: $migration_name"
}

restore_checkpoint() {
    local migration_name="$1"
    local checkpoint_name="$2"
    local checkpoint_file="$MIGRATION_STATE_DIR/${migration_name}_${checkpoint_name}.ckpt"

    if [ -f "$checkpoint_file" ]; then
        cat "$checkpoint_file"
        echo "[$(date)] Restored checkpoint: $checkpoint_name for migration: $migration_name"
    else
        echo "[$(date)] Checkpoint not found: $checkpoint_name" >&2
        return 1
    fi
}

🔧 Data Migration Patterns

Database Schema and Data Migration

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
# database-migration.sh - Database migration utilities

# Schema migration
migrate_database_schema() {
    local db_type="$1"
    local connection_string="$2"
    local migration_script="$3"
    local migration_version="$4"

    local current_version
    current_version=$(get_current_schema_version "$db_type" "$connection_string")

    if [ "$current_version" = "$migration_version" ]; then
        echo "[$(date)] Schema already at version $migration_version"
        return 0
    fi

    echo "[$(date)] Migrating schema from version $current_version to $migration_version"

    case "$db_type" in
        postgresql)
            migrate_postgresql_schema "$connection_string" "$migration_script"
            ;;
        mysql)
            migrate_mysql_schema "$connection_string" "$migration_script"
            ;;
        *)
            echo "[$(date)] Unsupported database type: $db_type" >&2
            return 1
            ;;
    esac
}

migrate_postgresql_schema() {
    local connection_string="$1"
    local migration_script="$2"

    # Backup current schema
    local backup_file="/tmp/schema_backup_$(date +%s).sql"
    if ! pg_dump --schema-only "$connection_string" > "$backup_file"; then
        echo "[$(date)] Failed to backup current schema" >&2
        return 1
    fi

    # Apply migration
    if psql "$connection_string" -f "$migration_script"; then
        echo "[$(date)] PostgreSQL schema migration completed successfully"
        # Update schema version tracking
        update_schema_version "postgresql" "$connection_string" "$migration_script"
        return 0
    else
        echo "[$(date)] PostgreSQL schema migration failed, attempting rollback" >&2
        # Rollback using backup
        if psql "$connection_string" -f "$backup_file"; then
            echo "[$(date)] Schema rollback successful"
        else
            echo "[$(date)] Schema rollback failed - manual intervention required" >&2
        fi
        return 1
    fi
}

migrate_mysql_schema() {
    local connection_string="$1"
    local migration_script="$2"

    # Parse connection string for mysql
    local host port user password database
    parse_mysql_connection_string "$connection_string"

    # Backup current schema
    local backup_file="/tmp/schema_backup_$(date +%s).sql"
    if ! mysqldump -h "$host" -P "$port" -u "$user" -p"$password" \
        --no-data "$database" > "$backup_file"; then
        echo "[$(date)] Failed to backup current schema" >&2
        return 1
    fi

    # Apply migration
    if mysql -h "$host" -P "$port" -u "$user" -p"$password" \
        "$database" < "$migration_script"; then
        echo "[$(date)] MySQL schema migration completed successfully"
        update_schema_version "mysql" "$connection_string" "$migration_script"
        return 0
    else
        echo "[$(date)] MySQL schema migration failed, attempting rollback" >&2
        if mysql -h "$host" -P "$port" -u "$user" -p"$password" \
            "$database" < "$backup_file"; then
            echo "[$(date)] Schema rollback successful"
        else
            echo "[$(date)] Schema rollback failed - manual intervention required" >&2
        fi
        return 1
    fi
}

# Data migration with batching
migrate_data_batch() {
    local source_db="$1"
    local target_db="$2"
    local table_name="$3"
    local batch_size="${4:-1000}"

    echo "[$(date)] Starting data migration for table: $table_name"

    # Get total row count
    local total_rows
    total_rows=$(get_table_row_count "$source_db" "$table_name")

    echo "[$(date)] Total rows to migrate: $total_rows"

    local offset=0
    local migrated_rows=0

    while [ $offset -lt $total_rows ]; do
        echo "[$(date)] Migrating batch: $offset - $((offset + batch_size))"

        # Extract batch
        if ! extract_data_batch "$source_db" "$table_name" "$offset" "$batch_size" "/tmp/batch_${offset}.sql"; then
            echo "[$(date)] Failed to extract batch at offset $offset" >&2
            return 1
        fi

        # Transform if needed
        if ! transform_data_batch "/tmp/batch_${offset}.sql" "/tmp/transformed_${offset}.sql"; then
            echo "[$(date)] Failed to transform batch at offset $offset" >&2
            return 1
        fi

        # Load batch
        if ! load_data_batch "$target_db" "/tmp/transformed_${offset}.sql"; then
            echo "[$(date)] Failed to load batch at offset $offset" >&2
            return 1
        fi

        # Update counters
        migrated_rows=$((migrated_rows + batch_size))
        offset=$((offset + batch_size))

        # Cleanup temporary files
        rm -f "/tmp/batch_${offset}.sql" "/tmp/transformed_${offset}.sql"

        # Report progress
        local progress
        progress=$((migrated_rows * 100 / total_rows))
        echo "[$(date)] Migration progress: $progress%"
    done

    echo "[$(date)] Data migration completed successfully"
    return 0
}

# Schema version tracking
get_current_schema_version() {
    local db_type="$1"
    local connection_string="$2"

    case "$db_type" in
        postgresql)
            echo "SELECT version FROM schema_migrations ORDER BY applied_at DESC LIMIT 1;" | \
            psql "$connection_string" -t -A 2>/dev/null || echo "0"
            ;;
        mysql)
            local host port user password database
            parse_mysql_connection_string "$connection_string"
            echo "SELECT version FROM schema_migrations ORDER BY applied_at DESC LIMIT 1;" | \
            mysql -h "$host" -P "$port" -u "$user" -p"$password" -N "$database" 2>/dev/null || echo "0"
            ;;
    esac
}

update_schema_version() {
    local db_type="$1"
    local connection_string="$2"
    local migration_script="$3"

    local version
    version=$(basename "$migration_script" | cut -d'_' -f1)
    local timestamp
    timestamp=$(date -u +"%Y-%m-%d %H:%M:%S")

    case "$db_type" in
        postgresql)
            echo "INSERT INTO schema_migrations (version, applied_at) VALUES ('$version', '$timestamp');" | \
            psql "$connection_string" 2>/dev/null
            ;;
        mysql)
            local host port user password database
            parse_mysql_connection_string "$connection_string"
            echo "INSERT INTO schema_migrations (version, applied_at) VALUES ('$version', '$timestamp');" | \
            mysql -h "$host" -P "$port" -u "$user" -p"$password" "$database" 2>/dev/null
            ;;
    esac
}

📦 File and Directory Migration

Bulk File Operations with Progress Tracking

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
# file-migration.sh - File and directory migration utilities

# Parallel file migration with progress tracking
migrate_files_parallel() {
    local source_dir="$1"
    local target_dir="$2"
    local max_concurrent="${3:-4}"
    local file_pattern="${4:-*}"

    echo "[$(date)] Starting parallel file migration"
    echo "[$(date)] Source: $source_dir"
    echo "[$(date)] Target: $target_dir"
    echo "[$(date)] Max concurrent: $max_concurrent"

    # Create target directory
    mkdir -p "$target_dir"

    # Get file list
    local file_list
    file_list=$(find "$source_dir" -name "$file_pattern" -type f)
    local total_files
    total_files=$(echo "$file_list" | wc -l)

    echo "[$(date)] Total files to migrate: $total_files"

    # Process files in parallel batches
    local current_batch=0
    local completed_files=0

    echo "$file_list" | while read -r file; do
        if [ -n "$file" ]; then
            # Wait if we've reached max concurrent processes
            while [ $(jobs -r | wc -l) -ge $max_concurrent ]; do
                sleep 1
            done

            # Start migration in background
            {
                migrate_single_file "$file" "$source_dir" "$target_dir"
                local result=$?

                if [ $result -eq 0 ]; then
                    echo "[$(date)] Completed: $file"
                else
                    echo "[$(date)] Failed: $file" >&2
                fi

                # Update global counter (this is simplified - real implementation would use shared memory)
                completed_files=$((completed_files + 1))
                local progress=$((completed_files * 100 / total_files))
                echo "[$(date)] Progress: $progress% ($completed_files/$total_files)"

            } &

            current_batch=$((current_batch + 1))
        fi
    done

    # Wait for all background jobs to complete
    wait

    echo "[$(date)] File migration completed"
}

migrate_single_file() {
    local file_path="$1"
    local source_base="$2"
    local target_base="$3"

    # Calculate relative path
    local relative_path
    relative_path=${file_path#$source_base}
    local target_path="$target_base$relative_path"

    # Create target directory
    mkdir -p "$(dirname "$target_path")"

    # Copy file with verification
    if cp "$file_path" "$target_path"; then
        # Verify copy integrity
        if cmp -s "$file_path" "$target_path"; then
            return 0
        else
            echo "[$(date)] File integrity check failed: $file_path" >&2
            rm -f "$target_path"  # Remove corrupted copy
            return 1
        fi
    else
        echo "[$(date)] File copy failed: $file_path" >&2
        return 1
    fi
}

# Incremental file synchronization
sync_files_incremental() {
    local source_dir="$1"
    local target_dir="$2"
    local sync_method="${3:-rsync}"

    case "$sync_method" in
        rsync)
            sync_files_rsync "$source_dir" "$target_dir"
            ;;
        custom)
            sync_files_custom "$source_dir" "$target_dir"
            ;;
        *)
            echo "[$(date)] Unsupported sync method: $sync_method" >&2
            return 1
            ;;
    esac
}

sync_files_rsync() {
    local source_dir="$1"
    local target_dir="$2"

    # Use rsync with appropriate flags
    rsync -avH --delete \
        --progress \
        --stats \
        "$source_dir/" "$target_dir/" \
        2>&1 | tee -a "$MIGRATION_LOG_DIR/rsync_$(date +%Y%m%d_%H%M%S).log"

    local exit_code=${PIPESTATUS[0]}

    if [ $exit_code -eq 0 ]; then
        echo "[$(date)] Rsync completed successfully"
        return 0
    else
        echo "[$(date)] Rsync failed with exit code: $exit_code" >&2
        return 1
    fi
}

sync_files_custom() {
    local source_dir="$1"
    local target_dir="$2"

    # Custom implementation using find and checksums
    find "$source_dir" -type f | while read -r source_file; do
        local relative_path=${source_file#$source_dir}
        local target_file="$target_dir$relative_path"

        # Check if file exists in target
        if [ -f "$target_file" ]; then
            # Compare modification times and checksums
            local source_mtime
            local target_mtime
            source_mtime=$(stat -f %m "$source_file" 2>/dev/null || stat -c %Y "$source_file" 2>/dev/null)
            target_mtime=$(stat -f %m "$target_file" 2>/dev/null || stat -c %Y "$target_file" 2>/dev/null)

            if [ "$source_mtime" -gt "$target_mtime" ]; then
                # Source is newer, copy it
                echo "[$(date)] Updating: $relative_path"
                mkdir -p "$(dirname "$target_file")"
                cp "$source_file" "$target_file"
            fi
        else
            # File doesn't exist in target, copy it
            echo "[$(date)] Adding: $relative_path"
            mkdir -p "$(dirname "$target_file")"
            cp "$source_file" "$target_file"
        fi
    done
}

🔄 Batch Job Management

Robust Batch Processing with Retry Logic

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
# batch-job-manager.sh - Batch job execution framework

# Batch job configuration
BATCH_MAX_RETRIES=3
BATCH_RETRY_DELAY=60  # seconds
BATCH_CONCURRENT_JOBS=4
BATCH_JOB_TIMEOUT=3600  # 1 hour

# Execute batch job with retry logic
execute_batch_job() {
    local job_name="$1"
    local job_command="$2"
    local max_retries="${3:-$BATCH_MAX_RETRIES}"

    local attempt=1
    local success=false

    echo "[$(date)] Starting batch job: $job_name (Attempt $attempt/$max_retries)"

    while [ $attempt -le $max_retries ] && [ "$success" = false ]; do
        # Execute job with timeout
        if timeout $BATCH_JOB_TIMEOUT bash -c "$job_command"; then
            echo "[$(date)] Batch job completed successfully: $job_name"
            success=true
        else
            local exit_code=$?
            echo "[$(date)] Batch job failed: $job_name (Exit code: $exit_code, Attempt: $attempt)"

            if [ $attempt -lt $max_retries ]; then
                echo "[$(date)] Waiting $BATCH_RETRY_DELAY seconds before retry..."
                sleep $BATCH_RETRY_DELAY
                attempt=$((attempt + 1))
                echo "[$(date)] Retrying batch job: $job_name (Attempt $attempt/$max_retries)"
            fi
        fi
    done

    if [ "$success" = true ]; then
        return 0
    else
        echo "[$(date)] Batch job failed after $max_retries attempts: $job_name" >&2
        return 1
    fi
}

# Concurrent batch job execution
execute_batch_jobs_concurrent() {
    local job_definitions_file="$1"
    local max_concurrent="${2:-$BATCH_CONCURRENT_JOBS}"

    echo "[$(date)] Starting concurrent batch jobs from: $job_definitions_file"

    # Read job definitions and execute concurrently
    local running_jobs=0

    while IFS='|' read -r job_name job_command max_retries || [ -n "$job_name" ]; do
        if [ -n "$job_name" ]; then
            # Wait if we've reached max concurrent jobs
            while [ $running_jobs -ge $max_concurrent ]; do
                sleep 5
                # Check for completed jobs
                running_jobs=$(jobs -r | wc -l)
            done

            # Start job in background
            {
                execute_batch_job "$job_name" "$job_command" "$max_retries"
                local job_result=$?

                if [ $job_result -eq 0 ]; then
                    echo "[$(date)] Job succeeded: $job_name"
                else
                    echo "[$(date)] Job failed: $job_name" >&2
                fi

            } &

            running_jobs=$((running_jobs + 1))
        fi
    done < "$job_definitions_file"

    # Wait for all jobs to complete
    echo "[$(date)] Waiting for all batch jobs to complete..."
    wait

    echo "[$(date)] All batch jobs completed"
}

# Batch job monitoring and reporting
monitor_batch_jobs() {
    local job_names=("$@")

    echo "[$(date)] Monitoring batch jobs: ${job_names[*]}"

    for job_name in "${job_names[@]}"; do
        local job_status
        job_status=$(get_job_status "$job_name")

        case "$job_status" in
            running)
                local job_pid
                job_pid=$(get_job_pid "$job_name")
                local cpu_usage
                cpu_usage=$(ps -p "$job_pid" -o %cpu= 2>/dev/null || echo "N/A")
                local memory_usage
                memory_usage=$(ps -p "$job_pid" -o %mem= 2>/dev/null || echo "N/A")

                echo "[$(date)] Job $job_name: RUNNING (PID: $job_pid, CPU: $cpu_usage%, MEM: $memory_usage%)"
                ;;
            completed)
                echo "[$(date)] Job $job_name: COMPLETED"
                ;;
            failed)
                echo "[$(date)] Job $job_name: FAILED"
                ;;
            *)
                echo "[$(date)] Job $job_name: UNKNOWN STATUS"
                ;;
        esac
    done
}

# Job status tracking
get_job_status() {
    local job_name="$1"
    local status_file="$MIGRATION_STATE_DIR/job_${job_name}.status"

    if [ -f "$status_file" ]; then
        cat "$status_file"
    else
        echo "unknown"
    fi
}

set_job_status() {
    local job_name="$1"
    local status="$2"
    local status_file="$MIGRATION_STATE_DIR/job_${job_name}.status"

    echo "$status" > "$status_file"
}

get_job_pid() {
    local job_name="$1"
    local pid_file="$MIGRATION_STATE_DIR/job_${job_name}.pid"

    if [ -f "$pid_file" ]; then
        cat "$pid_file"
    else
        echo ""
    fi
}

set_job_pid() {
    local job_name="$1"
    local pid="$2"
    local pid_file="$MIGRATION_STATE_DIR/job_${job_name}.pid"

    echo "$pid" > "$pid_file"
}

🎨 Advanced Migration Features

Zero-Downtime Migration Strategies

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
# zero-downtime-migration.sh - Zero downtime migration utilities

# Blue-green deployment pattern
perform_blue_green_migration() {
    local service_name="$1"
    local blue_environment="$2"
    local green_environment="$3"
    local current_active="$4"  # blue or green

    local new_active
    if [ "$current_active" = "blue" ]; then
        new_active="green"
        local new_env="$green_environment"
        local old_env="$blue_environment"
    else
        new_active="blue"
        local new_env="$blue_environment"
        local old_env="$green_environment"
    fi

    echo "[$(date)] Starting blue-green migration for $service_name"
    echo "[$(date)] Current active: $current_active"
    echo "[$(date)] Migrating to: $new_active"

    # Step 1: Deploy to inactive environment
    echo "[$(date)] Deploying to $new_active environment"
    if ! deploy_to_environment "$service_name" "$new_env"; then
        echo "[$(date)] Deployment to $new_active failed" >&2
        return 1
    fi

    # Step 2: Validate new environment
    echo "[$(date)] Validating $new_active environment"
    if ! validate_environment "$new_env"; then
        echo "[$(date)] Validation of $new_active failed" >&2
        rollback_environment "$old_env"
        return 1
    fi

    # Step 3: Switch traffic
    echo "[$(date)] Switching traffic to $new_active environment"
    if ! switch_traffic "$service_name" "$new_active"; then
        echo "[$(date)] Traffic switch failed, rolling back" >&2
        switch_traffic "$service_name" "$current_active"
        rollback_environment "$old_env"
        return 1
    fi

    # Step 4: Decommission old environment
    echo "[$(date)] Decommissioning $current_active environment"
    decommission_environment "$old_env"

    echo "[$(date)] Blue-green migration completed successfully"
    echo "[$(date)] New active environment: $new_active"

    return 0
}

# Canary deployment pattern
perform_canary_migration() {
    local service_name="$1"
    local new_version="$2"
    local canary_percentage="${3:-5}"  # Start with 5% traffic
    local ramp_up_interval="${4:-300}"  # 5 minutes between increases

    echo "[$(date)] Starting canary migration for $service_name to version $new_version"
    echo "[$(date)] Initial canary percentage: $canary_percentage%"

    # Deploy canary instances
    if ! deploy_canary_instances "$service_name" "$new_version"; then
        echo "[$(date)] Canary deployment failed" >&2
        return 1
    fi

    # Gradually increase traffic
    local current_percentage=$canary_percentage

    while [ $current_percentage -lt 100 ]; do
        echo "[$(date)] Setting canary traffic to ${current_percentage}%"

        if ! set_canary_traffic "$service_name" "$current_percentage"; then
            echo "[$(date)] Failed to set canary traffic to ${current_percentage}%" >&2
            return 1
        fi

        # Monitor for issues
        if ! monitor_canary_health "$service_name" "$current_percentage"; then
            echo "[$(date)] Canary health check failed, rolling back" >&2
            rollback_canary_migration "$service_name"
            return 1
        fi

        # Wait before next increase
        echo "[$(date)] Waiting $ramp_up_interval seconds before increasing traffic"
        sleep $ramp_up_interval

        # Increase percentage
        current_percentage=$((current_percentage + canary_percentage))

        # Cap at 100%
        if [ $current_percentage -gt 100 ]; then
            current_percentage=100
        fi
    done

    # Complete migration
    echo "[$(date)] Canary migration completed - 100% traffic routed to new version"
    finalize_canary_migration "$service_name" "$new_version"

    return 0
}

# Data migration with dual-write pattern
perform_dual_write_migration() {
    local source_system="$1"
    local target_system="$2"
    local migration_window_hours="${3:-24}"

    echo "[$(date)] Starting dual-write migration from $source_system to $target_system"
    echo "[$(date)] Migration window: $migration_window_hours hours"

    # Step 1: Enable dual-write
    echo "[$(date)] Enabling dual-write to $target_system"
    if ! enable_dual_write "$source_system" "$target_system"; then
        echo "[$(date)] Failed to enable dual-write" >&2
        return 1
    fi

    # Step 2: Backfill historical data
    echo "[$(date)] Starting historical data backfill"
    if ! backfill_historical_data "$source_system" "$target_system"; then
        echo "[$(date)] Historical data backfill failed" >&2
        disable_dual_write "$source_system" "$target_system"
        return 1
    fi

    # Step 3: Monitor consistency during migration window
    local migration_start
    migration_start=$(date +%s)
    local migration_end
    migration_end=$((migration_start + (migration_window_hours * 3600)))

    echo "[$(date)] Entering migration monitoring window"

    while [ $(date +%s) -lt $migration_end ]; do
        if ! verify_data_consistency "$source_system" "$target_system"; then
            echo "[$(date)] Data consistency check failed" >&2
            disable_dual_write "$source_system" "$target_system"
            return 1
        fi

        echo "[$(date)] Data consistency verified"
        sleep 300  # Check every 5 minutes
    done

    # Step 4: Switch to target system
    echo "[$(date)] Migration window complete, switching to target system"
    if ! switch_to_target_system "$source_system" "$target_system"; then
        echo "[$(date)] Failed to switch to target system" >&2
        return 1
    fi

    # Step 5: Disable dual-write
    echo "[$(date)] Disabling dual-write to $target_system"
    disable_dual_write "$source_system" "$target_system"

    echo "[$(date)] Dual-write migration completed successfully"
    return 0
}

🧾 Summary Best Practices

Migration and Batch Job Guidelines

  1. Phased Approach: Break large migrations into smaller, manageable phases
  2. Rollback Planning: Always plan for rollback before starting migration
  3. Thorough Testing: Test migrations in staging environments first
  4. Monitoring: Implement comprehensive monitoring during migrations
  5. Checkpointing: Use checkpoints for resumable operations
  6. Parallel Processing: Leverage parallelism for performance while managing resources
  7. Error Handling: Implement robust error handling and retry logic
  8. Progress Tracking: Provide real-time progress updates
  9. Data Validation: Verify data integrity throughout the process
  10. Zero Downtime: Use blue-green or canary deployments when possible

Sample Migration Configuration

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# migration-config-example.sh - Migration configuration template

# Database migration settings
export SOURCE_DB="postgresql://user:pass@source:5432/db"
export TARGET_DB="postgresql://user:pass@target:5432/db"
export SCHEMA_MIGRATION_DIR="/etc/migrations/schema"
export DATA_MIGRATION_BATCH_SIZE=1000

# File migration settings
export SOURCE_FILES="/data/old"
export TARGET_FILES="/data/new"
export FILE_MIGRATION_PARALLEL=8
export FILE_MIGRATION_PATTERN="*.dat"

# Batch job settings
export BATCH_MAX_RETRIES=3
export BATCH_RETRY_DELAY=60
export BATCH_CONCURRENT_JOBS=4
export BATCH_JOB_TIMEOUT=3600

# Zero-downtime settings
export CANARY_INITIAL_PERCENTAGE=5
export CANARY_RAMP_UP_INTERVAL=300
export BLUE_GREEN_VALIDATION_TIMEOUT=600

🧠 Complete Migration Script

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
#!/bin/bash
# complete-migration-system.sh - Production-ready migration framework

set -euo pipefail

# Source migration modules
source migration-framework.sh
source database-migration.sh
source file-migration.sh
source batch-job-manager.sh
source zero-downtime-migration.sh

# Main migration controller
main() {
    local migration_type="$1"
    shift

    case "$migration_type" in
        database-schema)
            execute_database_schema_migration "$@"
            ;;
        database-data)
            execute_database_data_migration "$@"
            ;;
        file-sync)
            execute_file_migration "$@"
            ;;
        batch-jobs)
            execute_batch_jobs "$@"
            ;;
        blue-green)
            execute_blue_green_migration "$@"
            ;;
        canary)
            execute_canary_migration "$@"
            ;;
        dual-write)
            execute_dual_write_migration "$@"
            ;;
        *)
            echo "Usage: $0 {database-schema|database-data|file-sync|batch-jobs|blue-green|canary|dual-write} [options]" >&2
            return 1
            ;;
    esac
}

# Execute database schema migration
execute_database_schema_migration() {
    local migration_script="$1"
    local db_type="$2"
    local connection_string="$3"

    init_migration "schema_$(basename "$migration_script")"

    if acquire_migration_lock "schema_migration"; then
        if migrate_database_schema "$db_type" "$connection_string" "$migration_script"; then
            set_migration_state "schema_$(basename "$migration_script")" "$STATE_COMPLETED"
        else
            set_migration_state "schema_$(basename "$migration_script")" "$STATE_FAILED"
            release_migration_lock "schema_migration"
            return 1
        fi
        release_migration_lock "schema_migration"
    else
        echo "[$(date)] Migration already in progress" >&2
        return 1
    fi
}

# Run main function
main "$@"

🧾 See Also