fixing deploy step
ci/woodpecker/push/woodpecker Pipeline failed
Details
ci/woodpecker/push/woodpecker Pipeline failed
Details
This commit is contained in:
parent
4e1e5f21cf
commit
03b209140d
|
@ -75,12 +75,12 @@ debug() {
|
||||||
cleanup() {
|
cleanup() {
|
||||||
local exit_code=$?
|
local exit_code=$?
|
||||||
|
|
||||||
if [[ -f "$LOCK_FILE" ]]; then
|
if [ -f "$LOCK_FILE" ]; then
|
||||||
debug "Removing deployment lock file"
|
debug "Removing deployment lock file"
|
||||||
rm -f "$LOCK_FILE"
|
rm -f "$LOCK_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ $exit_code -ne 0 && "$ROLLBACK_NEEDED" == "true" ]]; then
|
if [ $exit_code -ne 0 ] && [ "$ROLLBACK_NEEDED" = "true" ]; then
|
||||||
error "Deployment failed - attempting rollback..."
|
error "Deployment failed - attempting rollback..."
|
||||||
attempt_rollback
|
attempt_rollback
|
||||||
fi
|
fi
|
||||||
|
@ -98,14 +98,14 @@ retry_command() {
|
||||||
local description="$2"
|
local description="$2"
|
||||||
local attempt=1
|
local attempt=1
|
||||||
|
|
||||||
while [[ $attempt -le $MAX_RETRIES ]]; do
|
while [ $attempt -le $MAX_RETRIES ]; do
|
||||||
log "Attempt $attempt/$MAX_RETRIES: $description"
|
log "Attempt $attempt/$MAX_RETRIES: $description"
|
||||||
|
|
||||||
if eval "$cmd"; then
|
if eval "$cmd"; then
|
||||||
success "$description completed successfully"
|
success "$description completed successfully"
|
||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
if [[ $attempt -eq $MAX_RETRIES ]]; then
|
if [ $attempt -eq $MAX_RETRIES ]; then
|
||||||
error "$description failed after $MAX_RETRIES attempts"
|
error "$description failed after $MAX_RETRIES attempts"
|
||||||
return 1
|
return 1
|
||||||
else
|
else
|
||||||
|
@ -114,7 +114,7 @@ retry_command() {
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
((attempt++))
|
attempt=$((attempt + 1))
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -178,7 +178,7 @@ pre_flight_checks() {
|
||||||
get_current_image_id() {
|
get_current_image_id() {
|
||||||
if docker stack ps "${CI_REPO_NAME}" >/dev/null 2>&1; then
|
if docker stack ps "${CI_REPO_NAME}" >/dev/null 2>&1; then
|
||||||
OLD_IMAGE_HASH=$(docker stack ps "${CI_REPO_NAME}" --format "table {{.Image}}" | grep authelia | head -n1 || echo "")
|
OLD_IMAGE_HASH=$(docker stack ps "${CI_REPO_NAME}" --format "table {{.Image}}" | grep authelia | head -n1 || echo "")
|
||||||
if [[ -n "$OLD_IMAGE_HASH" ]]; then
|
if [ -n "$OLD_IMAGE_HASH" ]; then
|
||||||
debug "Current image for rollback: $OLD_IMAGE_HASH"
|
debug "Current image for rollback: $OLD_IMAGE_HASH"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
@ -186,7 +186,7 @@ get_current_image_id() {
|
||||||
|
|
||||||
# Rollback function
|
# Rollback function
|
||||||
attempt_rollback() {
|
attempt_rollback() {
|
||||||
if [[ -n "$OLD_IMAGE_HASH" && "$OLD_IMAGE_HASH" != "IMAGE" ]]; then
|
if [ -n "$OLD_IMAGE_HASH" ] && [ "$OLD_IMAGE_HASH" != "IMAGE" ]; then
|
||||||
warning "Attempting rollback to previous image: $OLD_IMAGE_HASH"
|
warning "Attempting rollback to previous image: $OLD_IMAGE_HASH"
|
||||||
|
|
||||||
# This would require a more complex rollback mechanism
|
# This would require a more complex rollback mechanism
|
||||||
|
@ -233,9 +233,9 @@ force_pull_latest_images() {
|
||||||
retry_command "docker pull $redis_image" "Redis image pull"
|
retry_command "docker pull $redis_image" "Redis image pull"
|
||||||
|
|
||||||
# Verify we have a new image hash
|
# Verify we have a new image hash
|
||||||
if [[ -n "$NEW_IMAGE_HASH" && "$NEW_IMAGE_HASH" != "$OLD_IMAGE_HASH" ]]; then
|
if [ -n "$NEW_IMAGE_HASH" ] && [ "$NEW_IMAGE_HASH" != "$OLD_IMAGE_HASH" ]; then
|
||||||
success "🔄 New image detected: $OLD_IMAGE_HASH → $NEW_IMAGE_HASH"
|
success "🔄 New image detected: $OLD_IMAGE_HASH → $NEW_IMAGE_HASH"
|
||||||
elif [[ -n "$NEW_IMAGE_HASH" ]]; then
|
elif [ -n "$NEW_IMAGE_HASH" ]; then
|
||||||
warning "⚠️ Same image hash detected: $NEW_IMAGE_HASH (this may be expected)"
|
warning "⚠️ Same image hash detected: $NEW_IMAGE_HASH (this may be expected)"
|
||||||
else
|
else
|
||||||
error "❌ Could not determine new image hash"
|
error "❌ Could not determine new image hash"
|
||||||
|
@ -254,13 +254,13 @@ get_container_diagnostics() {
|
||||||
local tasks
|
local tasks
|
||||||
tasks=$(docker service ps "${CI_REPO_NAME}_${service_name}" --format "{{.ID}}\t{{.Name}}\t{{.CurrentState}}\t{{.Error}}" --no-trunc)
|
tasks=$(docker service ps "${CI_REPO_NAME}_${service_name}" --format "{{.ID}}\t{{.Name}}\t{{.CurrentState}}\t{{.Error}}" --no-trunc)
|
||||||
|
|
||||||
if [[ -n "$tasks" ]]; then
|
if [ -n "$tasks" ]; then
|
||||||
error "Service tasks:"
|
error "Service tasks:"
|
||||||
echo "$tasks" | while IFS=$'\t' read -r task_id name state task_error; do
|
echo "$tasks" | while IFS=$'\t' read -r task_id name state task_error; do
|
||||||
error " Task: $name"
|
error " Task: $name"
|
||||||
error " ID: $task_id"
|
error " ID: $task_id"
|
||||||
error " State: $state"
|
error " State: $state"
|
||||||
if [[ -n "$task_error" ]]; then
|
if [ -n "$task_error" ]; then
|
||||||
error " Error: $task_error"
|
error " Error: $task_error"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -268,7 +268,7 @@ get_container_diagnostics() {
|
||||||
log "Attempting to get logs for task $task_id..."
|
log "Attempting to get logs for task $task_id..."
|
||||||
local task_logs
|
local task_logs
|
||||||
task_logs=$(docker service logs "${CI_REPO_NAME}_${service_name}" --raw --tail 20 2>/dev/null || echo "No logs available")
|
task_logs=$(docker service logs "${CI_REPO_NAME}_${service_name}" --raw --tail 20 2>/dev/null || echo "No logs available")
|
||||||
if [[ "$task_logs" != "No logs available" ]]; then
|
if [ "$task_logs" != "No logs available" ]; then
|
||||||
error " Recent logs:"
|
error " Recent logs:"
|
||||||
echo "$task_logs" | sed 's/^/ /'
|
echo "$task_logs" | sed 's/^/ /'
|
||||||
fi
|
fi
|
||||||
|
@ -285,7 +285,7 @@ get_container_diagnostics() {
|
||||||
local containers
|
local containers
|
||||||
containers=$(docker ps -a --filter "label=com.docker.swarm.service.name=${CI_REPO_NAME}_${service_name}" --format "{{.ID}}\t{{.Status}}\t{{.Names}}" 2>/dev/null || echo "")
|
containers=$(docker ps -a --filter "label=com.docker.swarm.service.name=${CI_REPO_NAME}_${service_name}" --format "{{.ID}}\t{{.Status}}\t{{.Names}}" 2>/dev/null || echo "")
|
||||||
|
|
||||||
if [[ -n "$containers" ]]; then
|
if [ -n "$containers" ]; then
|
||||||
error "Associated containers:"
|
error "Associated containers:"
|
||||||
echo "$containers" | while IFS=$'\t' read -r container_id status name; do
|
echo "$containers" | while IFS=$'\t' read -r container_id status name; do
|
||||||
error " Container: $name ($container_id)"
|
error " Container: $name ($container_id)"
|
||||||
|
@ -311,12 +311,12 @@ wait_for_stack_removal() {
|
||||||
local elapsed=0
|
local elapsed=0
|
||||||
|
|
||||||
while docker stack ls | grep -q "${CI_REPO_NAME}"; do
|
while docker stack ls | grep -q "${CI_REPO_NAME}"; do
|
||||||
if [[ $elapsed -ge $timeout ]]; then
|
if [ $elapsed -ge $timeout ]; then
|
||||||
error "Stack removal timeout after ${timeout} seconds"
|
error "Stack removal timeout after ${timeout} seconds"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ $((elapsed % 10)) -eq 0 ]]; then # Log every 10 seconds instead of 5
|
if [ $((elapsed % 10)) -eq 0 ]; then # Log every 10 seconds instead of 5
|
||||||
log "Stack still exists, waiting... (${elapsed}s/${timeout}s)"
|
log "Stack still exists, waiting... (${elapsed}s/${timeout}s)"
|
||||||
fi
|
fi
|
||||||
sleep 2 # Check every 2 seconds instead of 5
|
sleep 2 # Check every 2 seconds instead of 5
|
||||||
|
@ -410,12 +410,12 @@ comprehensive_health_check() {
|
||||||
local authelia_healthy=false
|
local authelia_healthy=false
|
||||||
local last_status=""
|
local last_status=""
|
||||||
|
|
||||||
while [[ $check_count -lt $max_checks ]]; do
|
while [ $check_count -lt $max_checks ]; do
|
||||||
local current_time=$(date +%s)
|
local current_time=$(date +%s)
|
||||||
local elapsed=$((current_time - start_time))
|
local elapsed=$((current_time - start_time))
|
||||||
|
|
||||||
# Only log every 10 seconds to reduce noise
|
# Only log every 10 seconds to reduce noise
|
||||||
if [[ $((check_count % 5)) -eq 0 ]]; then
|
if [ $((check_count % 5)) -eq 0 ]; then
|
||||||
log "Health check ${check_count}/${max_checks} (${elapsed}s elapsed)"
|
log "Health check ${check_count}/${max_checks} (${elapsed}s elapsed)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -423,7 +423,7 @@ comprehensive_health_check() {
|
||||||
local service_status
|
local service_status
|
||||||
service_status=$(docker stack ps "${CI_REPO_NAME}" --format "{{.Name}}\t{{.CurrentState}}\t{{.Error}}" | grep "authelia_authelia" | head -n1)
|
service_status=$(docker stack ps "${CI_REPO_NAME}" --format "{{.Name}}\t{{.CurrentState}}\t{{.Error}}" | grep "authelia_authelia" | head -n1)
|
||||||
|
|
||||||
if [[ -n "$service_status" ]]; then
|
if [ -n "$service_status" ]; then
|
||||||
local name=$(echo "$service_status" | cut -f1)
|
local name=$(echo "$service_status" | cut -f1)
|
||||||
local state=$(echo "$service_status" | cut -f2)
|
local state=$(echo "$service_status" | cut -f2)
|
||||||
local error_msg=$(echo "$service_status" | cut -f3)
|
local error_msg=$(echo "$service_status" | cut -f3)
|
||||||
|
@ -431,17 +431,17 @@ comprehensive_health_check() {
|
||||||
# Check for Running state
|
# Check for Running state
|
||||||
if echo "$state" | grep -q "Running"; then
|
if echo "$state" | grep -q "Running"; then
|
||||||
# Verify it's actually stable by checking for a few seconds
|
# Verify it's actually stable by checking for a few seconds
|
||||||
if [[ "$last_status" == "Running" ]]; then
|
if [ "$last_status" = "Running" ]; then
|
||||||
# Double-check: no recent failures
|
# Double-check: no recent failures
|
||||||
local failed_count
|
local failed_count
|
||||||
failed_count=$(docker stack ps "${CI_REPO_NAME}" | grep "authelia_authelia" | grep -c "Failed" || echo "0")
|
failed_count=$(docker stack ps "${CI_REPO_NAME}" | grep "authelia_authelia" | grep -c "Failed" || echo "0")
|
||||||
|
|
||||||
if [[ $failed_count -eq 0 ]]; then
|
if [ $failed_count -eq 0 ]; then
|
||||||
# Final verification: ensure we're using the new image
|
# Final verification: ensure we're using the new image
|
||||||
local current_image
|
local current_image
|
||||||
current_image=$(docker stack ps "${CI_REPO_NAME}" --format "{{.Image}}" | grep authelia | head -n1)
|
current_image=$(docker stack ps "${CI_REPO_NAME}" --format "{{.Image}}" | grep authelia | head -n1)
|
||||||
|
|
||||||
if [[ "$current_image" == *"$NEW_IMAGE_HASH"* ]] || [[ -z "$NEW_IMAGE_HASH" ]]; then
|
if echo "$current_image" | grep -q "$NEW_IMAGE_HASH" || [ -z "$NEW_IMAGE_HASH" ]; then
|
||||||
success "✅ Authelia service is healthy and running!"
|
success "✅ Authelia service is healthy and running!"
|
||||||
success "🎯 Using correct image: $current_image"
|
success "🎯 Using correct image: $current_image"
|
||||||
success "⚡ Total deployment time: ${elapsed} seconds"
|
success "⚡ Total deployment time: ${elapsed} seconds"
|
||||||
|
@ -457,7 +457,7 @@ comprehensive_health_check() {
|
||||||
last_status="Running"
|
last_status="Running"
|
||||||
elif echo "$state" | grep -q "Failed\|Rejected\|Shutdown"; then
|
elif echo "$state" | grep -q "Failed\|Rejected\|Shutdown"; then
|
||||||
error "❌ Service failed: $state"
|
error "❌ Service failed: $state"
|
||||||
if [[ -n "$error_msg" ]]; then
|
if [ -n "$error_msg" ]; then
|
||||||
error "Error: $error_msg"
|
error "Error: $error_msg"
|
||||||
fi
|
fi
|
||||||
break # Exit early on clear failure
|
break # Exit early on clear failure
|
||||||
|
@ -467,12 +467,12 @@ comprehensive_health_check() {
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ $elapsed -ge $timeout ]]; then
|
if [ $elapsed -ge $timeout ]; then
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
|
||||||
sleep 2
|
sleep 2
|
||||||
((check_count++))
|
check_count=$((check_count + 1))
|
||||||
done
|
done
|
||||||
|
|
||||||
# Health check failed - provide comprehensive diagnostics
|
# Health check failed - provide comprehensive diagnostics
|
||||||
|
@ -482,8 +482,8 @@ comprehensive_health_check() {
|
||||||
# Get detailed diagnostics for each service
|
# Get detailed diagnostics for each service
|
||||||
log "🔍 Gathering comprehensive diagnostics..."
|
log "🔍 Gathering comprehensive diagnostics..."
|
||||||
|
|
||||||
local services=("authelia" "mariadb" "redis")
|
local services="authelia mariadb redis"
|
||||||
for service in "${services[@]}"; do
|
for service in $services; do
|
||||||
if docker service ls --format "{{.Name}}" | grep -q "${CI_REPO_NAME}_${service}"; then
|
if docker service ls --format "{{.Name}}" | grep -q "${CI_REPO_NAME}_${service}"; then
|
||||||
get_container_diagnostics "$service"
|
get_container_diagnostics "$service"
|
||||||
else
|
else
|
||||||
|
|
Loading…
Reference in New Issue