From 964563149684dc2361b0384c6fb31bc7939f3843 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 5 Jun 2025 09:06:59 -0400 Subject: [PATCH] fixing deploy step --- .woodpecker.yml | 39 +------- scripts/ci-deploy-production.sh | 153 ++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 38 deletions(-) create mode 100755 scripts/ci-deploy-production.sh diff --git a/.woodpecker.yml b/.woodpecker.yml index b1faefc..7a82b53 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -205,44 +205,7 @@ steps: volumes: - /var/run/docker.sock:/var/run/docker.sock commands: - - echo "Deploying to production environment" - - echo "$${REGISTRY_PASSWORD}" | docker login -u "$${REGISTRY_USER}" --password-stdin git.nixc.us - - echo "Removing old stack to release secrets" - - docker stack rm $${CI_REPO_NAME} || true - - echo "Waiting for complete stack removal (30 seconds)" - - sleep 30 - - echo "Verifying stack removal completed" - - while docker stack ls | grep -q $${CI_REPO_NAME}; do echo "Stack still exists, waiting..."; sleep 5; done - - echo "Removing old Docker secrets" - - docker secret rm AUTHENTICATION_BACKEND_LDAP_PASSWORD || true - - docker secret rm IDENTITY_VALIDATION_RESET_PASSWORD_JWT_SECRET || true - - docker secret rm STORAGE_ENCRYPTION_KEY || true - - docker secret rm SESSION_SECRET || true - - docker secret rm NOTIFIER_SMTP_PASSWORD || true - - docker secret rm IDENTITY_PROVIDERS_OIDC_HMAC_SECRET || true - - docker secret rm IDENTITY_PROVIDERS_OIDC_ISSUER_PRIVATE_KEY || true - - docker secret rm IDENTITY_PROVIDERS_OIDC_JWKS_KEY || true - - docker secret rm CLIENT_SECRET_HEADSCALE || true - - docker secret rm CLIENT_SECRET_HEADADMIN || true - - echo "Creating new Docker secrets with updated values" - - echo "$${AUTHENTICATION_BACKEND_LDAP_PASSWORD}" | docker secret create AUTHENTICATION_BACKEND_LDAP_PASSWORD - - - echo "$${IDENTITY_VALIDATION_RESET_PASSWORD_JWT_SECRET}" | docker secret create IDENTITY_VALIDATION_RESET_PASSWORD_JWT_SECRET - - - echo "$${STORAGE_ENCRYPTION_KEY}" | docker secret create STORAGE_ENCRYPTION_KEY - - - echo "$${SESSION_SECRET}" | docker secret create SESSION_SECRET - - - echo "$${NOTIFIER_SMTP_PASSWORD}" | docker secret create NOTIFIER_SMTP_PASSWORD - - - echo "$${IDENTITY_PROVIDERS_OIDC_HMAC_SECRET}" | docker secret create IDENTITY_PROVIDERS_OIDC_HMAC_SECRET - - - echo "$${IDENTITY_PROVIDERS_OIDC_ISSUER_PRIVATE_KEY}" | docker secret create IDENTITY_PROVIDERS_OIDC_ISSUER_PRIVATE_KEY - - - echo "$${IDENTITY_PROVIDERS_OIDC_JWKS_KEY}" | docker secret create IDENTITY_PROVIDERS_OIDC_JWKS_KEY - - - echo "$${CLIENT_SECRET_HEADSCALE}" | docker secret create CLIENT_SECRET_HEADSCALE - - - echo "$${CLIENT_SECRET_HEADADMIN}" | docker secret create CLIENT_SECRET_HEADADMIN - - - echo "Deploying new stack with fresh secrets" - - docker stack deploy --with-registry-auth -c ./stack.production.yml $${CI_REPO_NAME} - - echo "Waiting for services to initialize (30 seconds)" - - sleep 30 - - echo "Checking deployment status" - - docker stack ps $${CI_REPO_NAME} - - echo "Checking service health for 60 seconds" - - for i in {1..12}; do if docker stack ps $${CI_REPO_NAME} | grep Running | grep -q authelia_authelia; then echo "✅ Authelia service is running!"; break; elif [ $$i -eq 12 ]; then echo "❌ Deployment verification failed - showing logs:"; docker service logs $${CI_REPO_NAME}_authelia --tail 20; exit 1; else echo "Attempt $$i/12: Waiting for authelia service..."; sleep 5; fi; done + - ./scripts/ci-deploy-production.sh when: branch: main event: [push, cron] diff --git a/scripts/ci-deploy-production.sh b/scripts/ci-deploy-production.sh new file mode 100755 index 0000000..e539fe0 --- /dev/null +++ b/scripts/ci-deploy-production.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +################################################################################ +# WOODPECKER CI PRODUCTION DEPLOYMENT SCRIPT +################################################################################ +# +# ⚠️ WARNING: THIS SCRIPT IS EXCLUSIVELY FOR WOODPECKER CI USE +# +# This script is designed to run within the Woodpecker CI environment with +# specific environment variables and Docker socket access. +# +# 🚫 DO NOT RUN THIS ON A DEVELOPER WORKSTATION +# 🚫 This will attempt to remove production Docker stacks and secrets +# 🚫 This requires access to production Docker swarm manager nodes +# +# This script handles: +# - Production stack removal and cleanup +# - Docker secrets recreation with fresh values +# - New stack deployment with verification +# - Health checking and deployment validation +# +################################################################################ + +set -euo pipefail + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging function +log() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +error() { + echo -e "${RED}[ERROR] $1${NC}" +} + +success() { + echo -e "${GREEN}[SUCCESS] $1${NC}" +} + +warning() { + echo -e "${YELLOW}[WARNING] $1${NC}" +} + +# Verify we're running in CI environment +if [[ -z "${CI_REPO_NAME:-}" ]]; then + error "This script must only be run in Woodpecker CI environment!" + error "Missing CI_REPO_NAME environment variable" + exit 1 +fi + +log "Starting production deployment for ${CI_REPO_NAME}" + +# Step 1: Docker registry login +log "Logging into Docker registry" +echo "${REGISTRY_PASSWORD}" | docker login -u "${REGISTRY_USER}" --password-stdin git.nixc.us + +# Step 2: Remove old stack to release secrets +log "Removing old stack to release secrets" +docker stack rm "${CI_REPO_NAME}" || true + +# Step 3: Wait for complete stack removal +log "Waiting for complete stack removal (30 seconds)" +sleep 30 + +log "Verifying stack removal completed" +while docker stack ls | grep -q "${CI_REPO_NAME}"; do + log "Stack still exists, waiting..." + sleep 5 +done +success "Stack removal completed" + +# Step 4: Remove old Docker secrets +log "Removing old Docker secrets" +declare -a SECRETS=( + "AUTHENTICATION_BACKEND_LDAP_PASSWORD" + "IDENTITY_VALIDATION_RESET_PASSWORD_JWT_SECRET" + "STORAGE_ENCRYPTION_KEY" + "SESSION_SECRET" + "NOTIFIER_SMTP_PASSWORD" + "IDENTITY_PROVIDERS_OIDC_HMAC_SECRET" + "IDENTITY_PROVIDERS_OIDC_ISSUER_PRIVATE_KEY" + "IDENTITY_PROVIDERS_OIDC_JWKS_KEY" + "CLIENT_SECRET_HEADSCALE" + "CLIENT_SECRET_HEADADMIN" +) + +for secret in "${SECRETS[@]}"; do + docker secret rm "$secret" || true + log "Removed secret: $secret" +done + +# Step 5: Create new Docker secrets with updated values +log "Creating new Docker secrets with updated values" +for secret in "${SECRETS[@]}"; do + env_var="${secret}" + if [[ -n "${!env_var:-}" ]]; then + echo "${!env_var}" | docker secret create "$secret" - + success "Created secret: $secret" + else + error "Environment variable $env_var is not set!" + exit 1 + fi +done + +# Step 6: Deploy new stack with fresh secrets +log "Deploying new stack with fresh secrets" +docker stack deploy --with-registry-auth -c ./stack.production.yml "${CI_REPO_NAME}" + +# Step 7: Wait for services to initialize +log "Waiting for services to initialize (30 seconds)" +sleep 30 + +# Step 8: Check deployment status +log "Checking deployment status" +docker stack ps "${CI_REPO_NAME}" + +# Step 9: Health check loop for authelia service +log "Checking service health for 60 seconds" +for i in {1..12}; do + if docker stack ps "${CI_REPO_NAME}" | grep Running | grep -q "authelia_authelia"; then + success "✅ Authelia service is running!" + + # Additional health verification + log "Performing additional health checks..." + sleep 5 + + # Check if service is actually healthy (not just running) + if docker stack ps "${CI_REPO_NAME}" | grep -A 5 "authelia_authelia" | grep -q "Running"; then + success "🎉 Production deployment completed successfully!" + success "Authelia service is healthy and running" + exit 0 + fi + elif [ $i -eq 12 ]; then + error "❌ Deployment verification failed after 60 seconds" + error "Showing service logs for debugging:" + docker service logs "${CI_REPO_NAME}_authelia" --tail 20 + error "Showing stack status:" + docker stack ps "${CI_REPO_NAME}" + exit 1 + else + log "Attempt $i/12: Waiting for authelia service..." + sleep 5 + fi +done + +error "Health check timeout - this should not be reached" +exit 1 \ No newline at end of file