fixing deploy step
ci/woodpecker/push/woodpecker Pipeline failed
Details
ci/woodpecker/push/woodpecker Pipeline failed
Details
This commit is contained in:
parent
4e1e5f21cf
commit
03b209140d
|
@ -75,12 +75,12 @@ debug() {
|
|||
cleanup() {
|
||||
local exit_code=$?
|
||||
|
||||
if [[ -f "$LOCK_FILE" ]]; then
|
||||
if [ -f "$LOCK_FILE" ]; then
|
||||
debug "Removing deployment lock file"
|
||||
rm -f "$LOCK_FILE"
|
||||
fi
|
||||
|
||||
if [[ $exit_code -ne 0 && "$ROLLBACK_NEEDED" == "true" ]]; then
|
||||
if [ $exit_code -ne 0 ] && [ "$ROLLBACK_NEEDED" = "true" ]; then
|
||||
error "Deployment failed - attempting rollback..."
|
||||
attempt_rollback
|
||||
fi
|
||||
|
@ -98,14 +98,14 @@ retry_command() {
|
|||
local description="$2"
|
||||
local attempt=1
|
||||
|
||||
while [[ $attempt -le $MAX_RETRIES ]]; do
|
||||
while [ $attempt -le $MAX_RETRIES ]; do
|
||||
log "Attempt $attempt/$MAX_RETRIES: $description"
|
||||
|
||||
if eval "$cmd"; then
|
||||
success "$description completed successfully"
|
||||
return 0
|
||||
else
|
||||
if [[ $attempt -eq $MAX_RETRIES ]]; then
|
||||
if [ $attempt -eq $MAX_RETRIES ]; then
|
||||
error "$description failed after $MAX_RETRIES attempts"
|
||||
return 1
|
||||
else
|
||||
|
@ -114,7 +114,7 @@ retry_command() {
|
|||
fi
|
||||
fi
|
||||
|
||||
((attempt++))
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
}
|
||||
|
||||
|
@ -178,7 +178,7 @@ pre_flight_checks() {
|
|||
get_current_image_id() {
|
||||
if docker stack ps "${CI_REPO_NAME}" >/dev/null 2>&1; then
|
||||
OLD_IMAGE_HASH=$(docker stack ps "${CI_REPO_NAME}" --format "table {{.Image}}" | grep authelia | head -n1 || echo "")
|
||||
if [[ -n "$OLD_IMAGE_HASH" ]]; then
|
||||
if [ -n "$OLD_IMAGE_HASH" ]; then
|
||||
debug "Current image for rollback: $OLD_IMAGE_HASH"
|
||||
fi
|
||||
fi
|
||||
|
@ -186,7 +186,7 @@ get_current_image_id() {
|
|||
|
||||
# Rollback function
|
||||
attempt_rollback() {
|
||||
if [[ -n "$OLD_IMAGE_HASH" && "$OLD_IMAGE_HASH" != "IMAGE" ]]; then
|
||||
if [ -n "$OLD_IMAGE_HASH" ] && [ "$OLD_IMAGE_HASH" != "IMAGE" ]; then
|
||||
warning "Attempting rollback to previous image: $OLD_IMAGE_HASH"
|
||||
|
||||
# This would require a more complex rollback mechanism
|
||||
|
@ -233,9 +233,9 @@ force_pull_latest_images() {
|
|||
retry_command "docker pull $redis_image" "Redis image pull"
|
||||
|
||||
# Verify we have a new image hash
|
||||
if [[ -n "$NEW_IMAGE_HASH" && "$NEW_IMAGE_HASH" != "$OLD_IMAGE_HASH" ]]; then
|
||||
if [ -n "$NEW_IMAGE_HASH" ] && [ "$NEW_IMAGE_HASH" != "$OLD_IMAGE_HASH" ]; then
|
||||
success "🔄 New image detected: $OLD_IMAGE_HASH → $NEW_IMAGE_HASH"
|
||||
elif [[ -n "$NEW_IMAGE_HASH" ]]; then
|
||||
elif [ -n "$NEW_IMAGE_HASH" ]; then
|
||||
warning "⚠️ Same image hash detected: $NEW_IMAGE_HASH (this may be expected)"
|
||||
else
|
||||
error "❌ Could not determine new image hash"
|
||||
|
@ -254,13 +254,13 @@ get_container_diagnostics() {
|
|||
local tasks
|
||||
tasks=$(docker service ps "${CI_REPO_NAME}_${service_name}" --format "{{.ID}}\t{{.Name}}\t{{.CurrentState}}\t{{.Error}}" --no-trunc)
|
||||
|
||||
if [[ -n "$tasks" ]]; then
|
||||
if [ -n "$tasks" ]; then
|
||||
error "Service tasks:"
|
||||
echo "$tasks" | while IFS=$'\t' read -r task_id name state task_error; do
|
||||
error " Task: $name"
|
||||
error " ID: $task_id"
|
||||
error " State: $state"
|
||||
if [[ -n "$task_error" ]]; then
|
||||
if [ -n "$task_error" ]; then
|
||||
error " Error: $task_error"
|
||||
fi
|
||||
|
||||
|
@ -268,7 +268,7 @@ get_container_diagnostics() {
|
|||
log "Attempting to get logs for task $task_id..."
|
||||
local task_logs
|
||||
task_logs=$(docker service logs "${CI_REPO_NAME}_${service_name}" --raw --tail 20 2>/dev/null || echo "No logs available")
|
||||
if [[ "$task_logs" != "No logs available" ]]; then
|
||||
if [ "$task_logs" != "No logs available" ]; then
|
||||
error " Recent logs:"
|
||||
echo "$task_logs" | sed 's/^/ /'
|
||||
fi
|
||||
|
@ -285,7 +285,7 @@ get_container_diagnostics() {
|
|||
local containers
|
||||
containers=$(docker ps -a --filter "label=com.docker.swarm.service.name=${CI_REPO_NAME}_${service_name}" --format "{{.ID}}\t{{.Status}}\t{{.Names}}" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$containers" ]]; then
|
||||
if [ -n "$containers" ]; then
|
||||
error "Associated containers:"
|
||||
echo "$containers" | while IFS=$'\t' read -r container_id status name; do
|
||||
error " Container: $name ($container_id)"
|
||||
|
@ -311,12 +311,12 @@ wait_for_stack_removal() {
|
|||
local elapsed=0
|
||||
|
||||
while docker stack ls | grep -q "${CI_REPO_NAME}"; do
|
||||
if [[ $elapsed -ge $timeout ]]; then
|
||||
if [ $elapsed -ge $timeout ]; then
|
||||
error "Stack removal timeout after ${timeout} seconds"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ $((elapsed % 10)) -eq 0 ]]; then # Log every 10 seconds instead of 5
|
||||
if [ $((elapsed % 10)) -eq 0 ]; then # Log every 10 seconds instead of 5
|
||||
log "Stack still exists, waiting... (${elapsed}s/${timeout}s)"
|
||||
fi
|
||||
sleep 2 # Check every 2 seconds instead of 5
|
||||
|
@ -410,12 +410,12 @@ comprehensive_health_check() {
|
|||
local authelia_healthy=false
|
||||
local last_status=""
|
||||
|
||||
while [[ $check_count -lt $max_checks ]]; do
|
||||
while [ $check_count -lt $max_checks ]; do
|
||||
local current_time=$(date +%s)
|
||||
local elapsed=$((current_time - start_time))
|
||||
|
||||
# Only log every 10 seconds to reduce noise
|
||||
if [[ $((check_count % 5)) -eq 0 ]]; then
|
||||
if [ $((check_count % 5)) -eq 0 ]; then
|
||||
log "Health check ${check_count}/${max_checks} (${elapsed}s elapsed)"
|
||||
fi
|
||||
|
||||
|
@ -423,7 +423,7 @@ comprehensive_health_check() {
|
|||
local service_status
|
||||
service_status=$(docker stack ps "${CI_REPO_NAME}" --format "{{.Name}}\t{{.CurrentState}}\t{{.Error}}" | grep "authelia_authelia" | head -n1)
|
||||
|
||||
if [[ -n "$service_status" ]]; then
|
||||
if [ -n "$service_status" ]; then
|
||||
local name=$(echo "$service_status" | cut -f1)
|
||||
local state=$(echo "$service_status" | cut -f2)
|
||||
local error_msg=$(echo "$service_status" | cut -f3)
|
||||
|
@ -431,17 +431,17 @@ comprehensive_health_check() {
|
|||
# Check for Running state
|
||||
if echo "$state" | grep -q "Running"; then
|
||||
# Verify it's actually stable by checking for a few seconds
|
||||
if [[ "$last_status" == "Running" ]]; then
|
||||
if [ "$last_status" = "Running" ]; then
|
||||
# Double-check: no recent failures
|
||||
local failed_count
|
||||
failed_count=$(docker stack ps "${CI_REPO_NAME}" | grep "authelia_authelia" | grep -c "Failed" || echo "0")
|
||||
|
||||
if [[ $failed_count -eq 0 ]]; then
|
||||
if [ $failed_count -eq 0 ]; then
|
||||
# Final verification: ensure we're using the new image
|
||||
local current_image
|
||||
current_image=$(docker stack ps "${CI_REPO_NAME}" --format "{{.Image}}" | grep authelia | head -n1)
|
||||
|
||||
if [[ "$current_image" == *"$NEW_IMAGE_HASH"* ]] || [[ -z "$NEW_IMAGE_HASH" ]]; then
|
||||
if echo "$current_image" | grep -q "$NEW_IMAGE_HASH" || [ -z "$NEW_IMAGE_HASH" ]; then
|
||||
success "✅ Authelia service is healthy and running!"
|
||||
success "🎯 Using correct image: $current_image"
|
||||
success "⚡ Total deployment time: ${elapsed} seconds"
|
||||
|
@ -457,7 +457,7 @@ comprehensive_health_check() {
|
|||
last_status="Running"
|
||||
elif echo "$state" | grep -q "Failed\|Rejected\|Shutdown"; then
|
||||
error "❌ Service failed: $state"
|
||||
if [[ -n "$error_msg" ]]; then
|
||||
if [ -n "$error_msg" ]; then
|
||||
error "Error: $error_msg"
|
||||
fi
|
||||
break # Exit early on clear failure
|
||||
|
@ -467,12 +467,12 @@ comprehensive_health_check() {
|
|||
fi
|
||||
fi
|
||||
|
||||
if [[ $elapsed -ge $timeout ]]; then
|
||||
if [ $elapsed -ge $timeout ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
((check_count++))
|
||||
check_count=$((check_count + 1))
|
||||
done
|
||||
|
||||
# Health check failed - provide comprehensive diagnostics
|
||||
|
@ -482,8 +482,8 @@ comprehensive_health_check() {
|
|||
# Get detailed diagnostics for each service
|
||||
log "🔍 Gathering comprehensive diagnostics..."
|
||||
|
||||
local services=("authelia" "mariadb" "redis")
|
||||
for service in "${services[@]}"; do
|
||||
local services="authelia mariadb redis"
|
||||
for service in $services; do
|
||||
if docker service ls --format "{{.Name}}" | grep -q "${CI_REPO_NAME}_${service}"; then
|
||||
get_container_diagnostics "$service"
|
||||
else
|
||||
|
|
Loading…
Reference in New Issue