From 3aacfa8283be6b3b52bbd3a7d7f773b406a1107c Mon Sep 17 00:00:00 2001
From: Colin <colin@nixc.us>
Date: Sun, 12 Oct 2025 22:31:27 -0400
Subject: [PATCH] Add specialized fix scripts for macmini7 deployment and
 database issues

---
 fix_database.sh      | 76 +++++++++++++++++++++++++++++++++++++++++
 fix_from_macmini7.sh | 81 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 157 insertions(+)
 create mode 100755 fix_database.sh
 create mode 100755 fix_from_macmini7.sh

diff --git a/fix_database.sh b/fix_database.sh
new file mode 100755
index 0000000..5a052f5
--- /dev/null
+++ b/fix_database.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+# Script to fix database issues for Woodpecker
+
+echo "===== CHECKING DATABASE SERVICE ====="
+docker service ls | grep woodpecker_db || {
+  echo "ERROR: woodpecker_db service not found"
+  echo "Please deploy the stack first"
+  exit 1
+}
+
+echo "===== DATABASE SERVICE DETAILS ====="
+docker service ps woodpecker_db --no-trunc
+
+echo "===== DATABASE LOGS ====="
+docker service logs --tail 30 woodpecker_db
+
+echo "===== CHECKING DATABASE DIRECTORY ON MACMINI1 ====="
+echo "Note: This requires SSH access to macmini1 or shared storage"
+ls -la /mnt/data/nixc.us/woodpecker/production/db || {
+  echo "WARNING: Could not access database directory locally"
+  echo "Attempting to check via SSH..."
+  ssh macmini1 "ls -la /mnt/data/nixc.us/woodpecker/production/db" || {
+    echo "ERROR: Could not check database directory on macmini1 via SSH"
+  }
+}
+
+echo "===== FIXING DATABASE PERMISSIONS ====="
+echo "Setting permissions on database directory..."
+chmod -R 777 /mnt/data/nixc.us/woodpecker/production/db || {
+  echo "WARNING: Could not set permissions locally"
+  echo "Attempting to set permissions via SSH..."
+  ssh macmini1 "chmod -R 777 /mnt/data/nixc.us/woodpecker/production/db" || {
+    echo "ERROR: Could not set permissions on macmini1 via SSH"
+  }
+}
+
+echo "===== CHECKING FOR CORRUPTED DATABASE FILES ====="
+echo "Looking for InnoDB files that might be causing issues..."
+find /mnt/data/nixc.us/woodpecker/production/db -name "ib_logfile*" || {
+  echo "No InnoDB log files found locally"
+  echo "Attempting to check via SSH..."
+  ssh macmini1 "find /mnt/data/nixc.us/woodpecker/production/db -name \"ib_logfile*\"" || {
+    echo "ERROR: Could not check for InnoDB files on macmini1 via SSH"
+  }
+}
+
+echo "===== REMOVING CORRUPTED DATABASE FILES (IF ANY) ====="
+echo "This will only remove InnoDB log files that might be causing corruption"
+echo "Your actual database data will not be affected"
+echo "Removing InnoDB log files..."
+rm -f /mnt/data/nixc.us/woodpecker/production/db/ib_logfile* || {
+  echo "WARNING: Could not remove files locally"
+  echo "Attempting to remove via SSH..."
+  ssh macmini1 "rm -f /mnt/data/nixc.us/woodpecker/production/db/ib_logfile*" || {
+    echo "ERROR: Could not remove files on macmini1 via SSH"
+  }
+}
+
+echo "===== RESTARTING DATABASE SERVICE ====="
+docker service update --force woodpecker_db
+
+echo "===== WAITING FOR DATABASE TO RESTART ====="
+echo "Waiting 30 seconds for database to restart..."
+sleep 30
+
+echo "===== CHECKING DATABASE STATUS ====="
+docker service ps woodpecker_db --no-trunc
+
+echo "===== CHECKING DATABASE LOGS AFTER RESTART ====="
+docker service logs --tail 20 woodpecker_db
+
+echo "===== DONE ====="
+echo "If the database is still not running, you may need to try the following:"
+echo "1. Remove all files from /mnt/data/nixc.us/woodpecker/production/db"
+echo "2. Redeploy the stack"
+echo "This will reset your database, but will fix persistent corruption issues."
diff --git a/fix_from_macmini7.sh b/fix_from_macmini7.sh
new file mode 100755
index 0000000..eeb6ffd
--- /dev/null
+++ b/fix_from_macmini7.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+# Script to fix Woodpecker deployment from macmini7
+
+echo "===== VERIFYING WE'RE ON MACMINI7 ====="
+HOSTNAME=$(hostname)
+if [[ "$HOSTNAME" != *"macmini7"* ]]; then
+  echo "ERROR: This script should be run on macmini7, current host is $HOSTNAME"
+  echo "Please run this script on macmini7"
+  exit 1
+fi
+
+echo "===== CHECKING SWARM ACCESS FROM MACMINI7 ====="
+docker node ls || {
+  echo "ERROR: Cannot access Docker swarm from macmini7"
+  echo "Make sure macmini7 is a swarm manager"
+  exit 1
+}
+
+echo "===== CHECKING IF MACMINI1 IS IN SWARM ====="
+docker node ls | grep macmini1 || {
+  echo "WARNING: macmini1 node not found in swarm!"
+  echo "The stack.yml has constraints requiring macmini1"
+}
+
+echo "===== REMOVING ANY EXISTING WOODPECKER STACK ====="
+docker stack rm woodpecker
+echo "Waiting for stack removal..."
+sleep 20
+
+echo "===== CHECKING FOR ORPHANED SERVICES ====="
+docker service ls | grep woodpecker && echo "WARNING: Woodpecker services still exist after stack removal!"
+
+echo "===== ENSURING DIRECTORIES EXIST ON MACMINI1 ====="
+echo "Note: This requires SSH access to macmini1 or shared storage"
+echo "Creating directories for Woodpecker..."
+mkdir -p /mnt/data/nixc.us/woodpecker/production/db /mnt/data/nixc.us/woodpecker/production/data || {
+  echo "WARNING: Could not create directories locally"
+  echo "Attempting to create directories via SSH..."
+  ssh macmini1 "mkdir -p /mnt/data/nixc.us/woodpecker/production/db /mnt/data/nixc.us/woodpecker/production/data" || {
+    echo "ERROR: Could not create directories on macmini1 via SSH"
+    echo "You may need to manually create these directories on macmini1"
+  }
+}
+
+echo "===== ENSURING NETWORKS EXIST ====="
+docker network create --driver=overlay --attachable traefik || echo "traefik network already exists"
+
+echo "===== DEPLOYING WOODPECKER STACK FROM MACMINI7 ====="
+cd /mnt/tank/persist/nixc.us/woodpecker/production
+echo "Current directory: $(pwd)"
+ls -la stack.yml || {
+  echo "ERROR: stack.yml not found in $(pwd)"
+  echo "Please make sure the stack.yml file exists in this directory"
+  exit 1
+}
+
+echo "Deploying from stack.yml..."
+docker stack deploy -c stack.yml woodpecker
+
+echo "===== WAITING FOR SERVICES TO START ====="
+echo "Waiting 30 seconds for services to start..."
+sleep 30
+
+echo "===== SERVICE STATUS ====="
+docker service ls | grep woodpecker
+
+echo "===== CHECKING SERVICE PLACEMENT ====="
+for service in $(docker service ls --format "{{.Name}}" | grep woodpecker); do
+  echo "Checking placement for $service..."
+  docker service ps $service --no-trunc --format "{{.Node}} {{.CurrentState}}"
+done
+
+echo "===== CHECKING LOGS ====="
+echo "Database logs:"
+docker service logs --tail 20 woodpecker_db || echo "Could not get database logs"
+
+echo "Server logs:"
+docker service logs --tail 20 woodpecker_server || echo "Could not get server logs"
+
+echo "===== DONE ====="
+echo "If services are not running, check the logs above for errors."