haste/migrate-to-ingress.sh

#!/bin/bash
set -euo pipefail

# Migration script to move haste.nixc.us services from macmini3 to ingress.nixc.us
# This script uses non-interactive SSH and handles Docker volume migration

# Configuration
SOURCE_HOST="${SOURCE_HOST:-macmini3}"
TARGET_HOST="${TARGET_HOST:-ingress.nixc.us}"
STACK_NAME="${STACK_NAME:-haste}"
TARGET_HOSTNAME="${TARGET_HOSTNAME:-}"  # Will be auto-detected if not set
VOLUMES=("redis_data" "public_system")
BACKUP_DIR="/tmp/haste-migration-$(date +%Y%m%d-%H%M%S)"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# SSH helper function for non-interactive commands
ssh_cmd() {
    local host="$1"
    shift
    ssh -o BatchMode=yes -o ConnectTimeout=5 "$host" "$@"
}

# Check if SSH key is available
check_ssh() {
    log_info "Checking SSH connectivity..."
    if ! ssh_cmd "${SOURCE_HOST}" "echo 'SSH to source OK'" 2>/dev/null; then
        log_error "Cannot connect to ${SOURCE_HOST} via SSH. Please ensure SSH keys are set up."
        exit 1
    fi
    if ! ssh_cmd "${TARGET_HOST}" "echo 'SSH to target OK'" 2>/dev/null; then
        log_error "Cannot connect to ${TARGET_HOST} via SSH. Please ensure SSH keys are set up."
        exit 1
    fi
    log_info "SSH connectivity verified"
}

# Backup volumes from source host
backup_volumes() {
    log_info "Backing up volumes from ${SOURCE_HOST}..."
    
    ssh_cmd "${SOURCE_HOST}" "mkdir -p ${BACKUP_DIR}"
    
    for volume in "${VOLUMES[@]}"; do
        log_info "Backing up volume: ${volume}"
        
        # Docker Swarm prefixes volumes with stack name
        VOLUME_NAME="${STACK_NAME}_${volume}"
        
        # Check if volume exists
        if ! ssh_cmd "${SOURCE_HOST}" "docker volume inspect ${VOLUME_NAME} >/dev/null 2>&1"; then
            log_warn "Volume ${VOLUME_NAME} not found, skipping..."
            continue
        fi
        
        # Create backup using a temporary container
        ssh_cmd "${SOURCE_HOST}" "docker run --rm -v ${VOLUME_NAME}:/source:ro -v ${BACKUP_DIR}:/backup alpine:latest tar czf /backup/${volume}.tar.gz -C /source ."
        
        if [ $? -eq 0 ]; then
            log_info "Successfully backed up ${volume}"
        else
            log_error "Failed to backup ${volume}"
            exit 1
        fi
    done
    
    log_info "Volume backup completed"
}

# Transfer backups to target host
transfer_backups() {
    log_info "Transferring backups to ${TARGET_HOST}..."
    
    ssh_cmd "${TARGET_HOST}" "mkdir -p ${BACKUP_DIR}"
    
    for volume in "${VOLUMES[@]}"; do
        log_info "Transferring ${volume}.tar.gz..."
        ssh_cmd "${SOURCE_HOST}" "cat ${BACKUP_DIR}/${volume}.tar.gz" | \
            ssh_cmd "${TARGET_HOST}" "cat > ${BACKUP_DIR}/${volume}.tar.gz"
        
        if [ $? -eq 0 ]; then
            log_info "Successfully transferred ${volume}"
        else
            log_error "Failed to transfer ${volume}"
            exit 1
        fi
    done
    
    log_info "Backup transfer completed"
}

# Restore volumes on target host
restore_volumes() {
    log_info "Restoring volumes on ${TARGET_HOST}..."
    
    for volume in "${VOLUMES[@]}"; do
        log_info "Restoring volume: ${volume}"
        
        # Docker Swarm prefixes volumes with stack name
        VOLUME_NAME="${STACK_NAME}_${volume}"
        
        # Create volume if it doesn't exist
        ssh_cmd "${TARGET_HOST}" "docker volume create ${VOLUME_NAME}" || true
        
        # Restore backup using a temporary container
        ssh_cmd "${TARGET_HOST}" "docker run --rm -v ${VOLUME_NAME}:/target -v ${BACKUP_DIR}:/backup alpine:latest sh -c 'rm -rf /target/* && tar xzf /backup/${volume}.tar.gz -C /target'"
        
        if [ $? -eq 0 ]; then
            log_info "Successfully restored ${volume}"
        else
            log_error "Failed to restore ${volume}"
            exit 1
        fi
    done
    
    log_info "Volume restoration completed"
}

# Update stack.production.yml to use new hostname
update_stack_config() {
    log_info "Updating stack.production.yml..."
    
    # Determine target hostname if not already set
    if [ -z "${TARGET_HOSTNAME}" ]; then
        # Try to get the actual hostname from the target
        TARGET_HOSTNAME=$(ssh_cmd "${TARGET_HOST}" "hostname" 2>/dev/null || echo "ingress")
        log_info "Target hostname auto-detected: ${TARGET_HOSTNAME}"
    else
        log_info "Using specified target hostname: ${TARGET_HOSTNAME}"
    fi
    
    # Update the stack file
    if [[ "$OSTYPE" == "darwin"* ]]; then
        # macOS uses BSD sed
        sed -i.bak "s/node.hostname == macmini3/node.hostname == ${TARGET_HOSTNAME}/g" stack.production.yml
    else
        # Linux uses GNU sed
        sed -i.bak "s/node.hostname == macmini3/node.hostname == ${TARGET_HOSTNAME}/g" stack.production.yml
    fi
    
    log_info "Stack configuration updated"
    log_warn "Backup of original file saved as stack.production.yml.bak"
}

# Deploy to new location
deploy_to_target() {
    log_info "Deploying stack to ${TARGET_HOST}..."
    
    # Copy updated stack file to target
    scp -o BatchMode=yes stack.production.yml "${TARGET_HOST}:/tmp/stack.production.yml"
    
    # Deploy on target (assuming Docker Swarm manager is accessible)
    ssh_cmd "${TARGET_HOST}" "docker stack deploy --with-registry-auth -c /tmp/stack.production.yml ${STACK_NAME}"
    
    if [ $? -eq 0 ]; then
        log_info "Stack deployed successfully"
    else
        log_error "Failed to deploy stack"
        exit 1
    fi
}

# Verify deployment
verify_deployment() {
    log_info "Verifying deployment..."
    
    sleep 10  # Give services time to start
    
    ssh_cmd "${TARGET_HOST}" "docker stack services ${STACK_NAME} --format 'table {{.Name}}\t{{.Replicas}}\t{{.Image}}'"
    
    log_info "Deployment verification completed"
}

# Optional cleanup function
do_cleanup_old() {
    if [ "${CLEANUP_OLD:-false}" = "true" ]; then
        log_info "Removing old deployment from ${SOURCE_HOST}..."
        ssh_cmd "${SOURCE_HOST}" "docker stack rm ${STACK_NAME}" || log_warn "Stack removal failed or already removed"
        sleep 5
        ssh_cmd "${SOURCE_HOST}" "docker volume rm ${STACK_NAME}_redis_data ${STACK_NAME}_public_system" || log_warn "Volume removal failed or already removed"
        log_info "Old deployment cleanup completed"
    else
        log_warn "Skipping cleanup of old deployment on ${SOURCE_HOST}"
        log_warn "To remove old deployment, run manually:"
        log_warn "  ssh ${SOURCE_HOST} 'docker stack rm ${STACK_NAME}'"
        log_warn "  ssh ${SOURCE_HOST} 'docker volume rm ${STACK_NAME}_redis_data ${STACK_NAME}_public_system'"
        log_warn ""
        log_warn "Or set CLEANUP_OLD=true environment variable to auto-cleanup"
    fi
}

# Cleanup temporary files
cleanup_temp_files() {
    log_info "Cleaning up temporary files..."
    ssh_cmd "${SOURCE_HOST}" "rm -rf ${BACKUP_DIR}" || true
    ssh_cmd "${TARGET_HOST}" "rm -rf ${BACKUP_DIR}" || true
    log_info "Cleanup completed"
}

# Main execution
main() {
    log_info "Starting migration from ${SOURCE_HOST} to ${TARGET_HOST}"
    
    check_ssh
    backup_volumes
    transfer_backups
    restore_volumes
    update_stack_config
    deploy_to_target
    verify_deployment
    cleanup_temp_files
    do_cleanup_old
    
    log_info "Migration completed successfully!"
    log_warn "Remember to:"
    log_warn "  1. Test the service at https://haste.nixc.us"
    log_warn "  2. Commit the updated stack.production.yml"
    log_warn "  3. Clean up old deployment on ${SOURCE_HOST} when ready"
}

# Run main function
main "$@"