bump

README.md

@@ -113,9 +113,11 @@ docker run -d \
 | `TRAEFIK_SSH_HOST` | Traefik host to SSH into **(required)** | - |
 | `TRAEFIK_SSH_USER` | SSH user on the Traefik host | `root` |
 | `TRAEFIK_SSH_KEY` | SSH key for Traefik host (path or PEM) **(required)** | - |
-| `TRAEFIK_CONFIG_DIR` | Remote path for dynamic configs | `/root/traefik/dynamic` |
 | `TRAEFIK_ENTRYPOINT` | Traefik entrypoint name | `websecure` |
 | `TRAEFIK_CERT_RESOLVER` | Traefik TLS cert resolver | `letsencryptresolver` |
+| `PURGE_STALE_LABELS_ON_START` | Remove all managed tunnel labels during startup; normally leave disabled so routes survive restarts | `false` |
+| `HEALTHCHECK_ADDR` | TCP address checked by `tunnel-server healthcheck` | `127.0.0.1:$SSH_PORT` |
+| `HEALTHCHECK_TIMEOUT_SECONDS` | Healthcheck TCP dial timeout | `2` |
 
 ### Client
 
@@ -247,6 +249,27 @@ If you run the tunnel server without Docker:
 
 ## Troubleshooting
 
+### Self-healing after ingress restarts
+
+The tunnel-server keeps Traefik tunnel labels across normal restarts and
+reconciles them on startup instead of deleting every route. Tunnel ports are
+chosen from a stable hash of the domain, so a reconnecting client gets the same
+port and preserved labels point back to the right listener once the client
+returns.
+
+Use a non-interactive healthcheck to verify the SSH ingress port:
+
+```bash
+tunnel-server healthcheck
+```
+
+For Swarm deployments, configure the service healthcheck to run the same command
+inside the container. The production stack example includes this healthcheck
+without requiring a Dockerfile change.
+
+If you intentionally need to clear every managed Traefik tunnel route, start the
+server with `PURGE_STALE_LABELS_ON_START=true` once, then disable it again.
+
 ### "unable to authenticate, attempted methods [none publickey]"
 
 This means the tunnel-server rejected the client's key. There are exactly two causes:
@@ -263,24 +286,14 @@ ssh root@ingress.nixc.us "cat /home/tunnel/.ssh/authorized_keys"
 ssh root@ingress.nixc.us "cat >> /home/tunnel/.ssh/authorized_keys" < YOUR_CLIENT_KEY.pub
 ```
 
-**2. The tunnel-server was not restarted after adding the key.**
-
-The tunnel-server loads `authorized_keys` into memory **once at startup**. Editing the file on disk does nothing until the container restarts. After adding a key, always restart:
-
-```bash
-ssh root@ingress.nixc.us "docker service update --force better-argo-tunnels_tunnel-server"
-```
-
-Then restart the client so it reconnects immediately instead of waiting for its backoff timer.
-
 ### Checklist for a new tunnel client
 
 1. Generate an ed25519 key (or reuse an existing one).
 2. Append the `.pub` to `/home/tunnel/.ssh/authorized_keys` on `ingress.nixc.us`.
-3. Restart the tunnel-server: `docker service update --force better-argo-tunnels_tunnel-server`.
+3. Start the client container with the private key mounted and `TUNNEL_KEY` pointing at it.
-4. Start the client container with the private key mounted and `TUNNEL_KEY` pointing at it.
 
-If all four steps happen, the tunnel will connect. If any step is skipped, it won't.
+The tunnel-server reloads `authorized_keys` on each auth attempt, so new keys do
+not require a server restart.
 
 ## Security Notes
 

archive/stack.production.yml

@@ -33,6 +33,12 @@ services:
         published: 2222
         protocol: tcp
         mode: host
+    healthcheck:
+      test: ["CMD", "tunnel-server", "healthcheck"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 10s
     deploy:
       replicas: 1
       placement:

cmd/server/main.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"log"
+	"net"
 	"os"
 	"os/signal"
 	"strconv"
 	"syscall"
+	"time"
 
 	"github.com/nixc/reverse-ssh-traefik/internal/buildinfo"
 	"github.com/nixc/reverse-ssh-traefik/internal/server"
@@ -39,8 +41,27 @@ func envInt(key string, fallback int) int {
 	return n
 }
 
+func envBool(key string, fallback bool) bool {
+	v := os.Getenv(key)
+	if v == "" {
+		return fallback
+	}
+	b, err := strconv.ParseBool(v)
+	if err != nil {
+		log.Printf("WARN: invalid %s=%q, using default %t", key, v, fallback)
+		return fallback
+	}
+	return b
+}
+
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
+
+	if len(os.Args) > 1 && os.Args[1] == "healthcheck" {
+		runHealthcheck()
+		return
+	}
+
 	log.Printf("tunnel-server starting commit=%s", buildinfo.Commit)
 
 	// SSH server config (for accepting tunnel clients).
@@ -79,9 +100,13 @@ func main() {
 	}
 	defer labels.Close()
 
-	// Purge stale tunnel labels left by a previous container instance.
+	if envBool("PURGE_STALE_LABELS_ON_START", false) {
-	if err := labels.PurgeAllTunnelLabels(); err != nil {
+		log.Printf("PURGE_STALE_LABELS_ON_START enabled; removing all tunnel labels")
-		log.Printf("WARN: failed to purge stale labels: %v", err)
+		if err := labels.PurgeAllTunnelLabels(); err != nil {
+			log.Printf("WARN: failed to purge stale labels: %v", err)
+		}
+	} else if err := labels.ReconcileExistingTunnelLabels(); err != nil {
+		log.Printf("WARN: failed to reconcile existing tunnel labels: %v", err)
 	}
 
 	// Initialize SSH server for tunnel clients.
@@ -105,3 +130,16 @@ func main() {
 		log.Fatalf("SSH server error: %v", err)
 	}
 }
+
+func runHealthcheck() {
+	sshPort := envOr("SSH_PORT", "2222")
+	addr := envOr("HEALTHCHECK_ADDR", "127.0.0.1:"+sshPort)
+	timeout := time.Duration(envInt("HEALTHCHECK_TIMEOUT_SECONDS", 2)) * time.Second
+
+	conn, err := net.DialTimeout("tcp", addr, timeout)
+	if err != nil {
+		log.Printf("healthcheck failed for %s: %v", addr, err)
+		os.Exit(1)
+	}
+	conn.Close()
+}

internal/server/labels.go

@@ -18,9 +18,9 @@ type LabelManager struct {
 	remoteHost   string // Swarm manager, e.g. "ingress.nixc.us"
 	remoteUser   string // SSH user
 	signer       ssh.Signer
-	serviceName  string // Swarm service name, e.g. "better-argo-tunnels_tunnel-server"
+	serviceName  string          // Swarm service name, e.g. "better-argo-tunnels_tunnel-server"
-	entrypoint   string // e.g. "websecure"
+	entrypoint   string          // e.g. "websecure"
-	certResolver string // e.g. "letsencryptresolver"
+	certResolver string          // e.g. "letsencryptresolver"
 	labels       map[string]bool // track which tunnel keys we've added
 	authLabels   map[string]bool // track which tunnel keys have auth middleware
 }
@@ -57,50 +57,42 @@ func NewLabelManager(
 	return lm, nil
 }
 
+// ReconcileExistingTunnelLabels adopts existing tunnel-* Traefik labels from
+// the Swarm service so a restarted server preserves routes while clients
+// reconnect. It does not remove labels; explicit purge remains opt-in.
+func (lm *LabelManager) ReconcileExistingTunnelLabels() error {
+	labels, err := lm.inspectLabels()
+	if err != nil {
+		return err
+	}
+
+	tunnelKeys, authKeys := collectExistingTunnelKeys(labels)
+
+	lm.mu.Lock()
+	for key := range tunnelKeys {
+		lm.labels[key] = true
+	}
+	for key := range authKeys {
+		lm.authLabels[key] = true
+	}
+	lm.mu.Unlock()
+
+	log.Printf("Reconciled %d existing tunnel route(s), %d with auth", len(tunnelKeys), len(authKeys))
+	return nil
+}
+
 // PurgeAllTunnelLabels removes every tunnel-* Traefik label from the Swarm
-// service. Called on startup so stale labels from a previous instance don't
+// service. This is intentionally opt-in because purging on every restart
-// route traffic to ports that no longer exist.
+// creates a 404 window until every client reconnects.
 func (lm *LabelManager) PurgeAllTunnelLabels() error {
-	addr := lm.remoteHost
+	labels, err := lm.inspectLabels()
-	if !strings.Contains(addr, ":") {
-		addr = addr + ":22"
-	}
-
-	config := &ssh.ClientConfig{
-		User:            lm.remoteUser,
-		Auth:            []ssh.AuthMethod{ssh.PublicKeys(lm.signer)},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-	}
-
-	client, err := ssh.Dial("tcp", addr, config)
 	if err != nil {
-		return fmt.Errorf("SSH dial %s: %w", addr, err)
+		return err
-	}
-	defer client.Close()
-
-	session, err := client.NewSession()
-	if err != nil {
-		return fmt.Errorf("SSH session: %w", err)
-	}
-	defer session.Close()
-
-	inspectCmd := fmt.Sprintf(
-		"docker service inspect --format '{{json .Spec.Labels}}' %s", lm.serviceName)
-	output, err := session.CombinedOutput(inspectCmd)
-	if err != nil {
-		return fmt.Errorf("inspect labels: %w (%s)", err, string(output))
-	}
-
-	var labels map[string]string
-	if err := json.Unmarshal(output, &labels); err != nil {
-		return fmt.Errorf("parse labels JSON: %w", err)
 	}
 
 	var rmLabels []string
 	for key := range labels {
-		if strings.HasPrefix(key, "traefik.http.routers.tunnel-") ||
+		if isTunnelLabel(key) {
-			strings.HasPrefix(key, "traefik.http.services.tunnel-") ||
-			strings.HasPrefix(key, "traefik.http.middlewares.tunnel-") {
 			rmLabels = append(rmLabels, key)
 		}
 	}
@@ -116,10 +108,89 @@ func (lm *LabelManager) PurgeAllTunnelLabels() error {
 		return fmt.Errorf("purge tunnel labels: %w", err)
 	}
 
-	log.Printf("Purged %d stale tunnel labels on startup", len(rmLabels))
+	log.Printf("Purged %d tunnel label(s)", len(rmLabels))
 	return nil
 }
 
+func (lm *LabelManager) inspectLabels() (map[string]string, error) {
+	addr := lm.remoteHost
+	if !strings.Contains(addr, ":") {
+		addr = addr + ":22"
+	}
+
+	config := &ssh.ClientConfig{
+		User:            lm.remoteUser,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(lm.signer)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+
+	client, err := ssh.Dial("tcp", addr, config)
+	if err != nil {
+		return nil, fmt.Errorf("SSH dial %s: %w", addr, err)
+	}
+	defer client.Close()
+
+	session, err := client.NewSession()
+	if err != nil {
+		return nil, fmt.Errorf("SSH session: %w", err)
+	}
+	defer session.Close()
+
+	inspectCmd := fmt.Sprintf(
+		"docker service inspect --format '{{json .Spec.Labels}}' %s", lm.serviceName)
+	output, err := session.CombinedOutput(inspectCmd)
+	if err != nil {
+		return nil, fmt.Errorf("inspect labels: %w (%s)", err, string(output))
+	}
+
+	var labels map[string]string
+	if err := json.Unmarshal(output, &labels); err != nil {
+		return nil, fmt.Errorf("parse labels JSON: %w", err)
+	}
+
+	return labels, nil
+}
+
+func isTunnelLabel(key string) bool {
+	return strings.HasPrefix(key, "traefik.http.routers.tunnel-") ||
+		strings.HasPrefix(key, "traefik.http.services.tunnel-") ||
+		strings.HasPrefix(key, "traefik.http.middlewares.tunnel-")
+}
+
+func collectExistingTunnelKeys(labels map[string]string) (map[string]bool, map[string]bool) {
+	tunnelKeys := make(map[string]bool)
+	authKeys := make(map[string]bool)
+
+	for label := range labels {
+		if key, ok := tunnelKeyFromLabel(label, "traefik.http.routers.tunnel-", "-router."); ok {
+			tunnelKeys[key] = true
+			continue
+		}
+		if key, ok := tunnelKeyFromLabel(label, "traefik.http.services.tunnel-", "-service."); ok {
+			tunnelKeys[key] = true
+			continue
+		}
+		if key, ok := tunnelKeyFromLabel(label, "traefik.http.middlewares.tunnel-", "-auth."); ok {
+			tunnelKeys[key] = true
+			authKeys[key] = true
+		}
+	}
+
+	return tunnelKeys, authKeys
+}
+
+func tunnelKeyFromLabel(label, prefix, suffix string) (string, bool) {
+	if !strings.HasPrefix(label, prefix) {
+		return "", false
+	}
+	rest := strings.TrimPrefix(label, prefix)
+	idx := strings.Index(rest, suffix)
+	if idx <= 0 {
+		return "", false
+	}
+	return rest[:idx], true
+}
+
 // Add adds Traefik routing labels to the Swarm service for a tunnel.
 // If authUser and authPass are non-empty, a basicauth middleware is also added.
 func (lm *LabelManager) Add(tunKey, domain string, port int, authUser, authPass string) error {

internal/server/labels_test.go

@@ -0,0 +1,39 @@
+package server
+
+import "testing"
+
+func TestCollectExistingTunnelKeys(t *testing.T) {
+	labels := map[string]string{
+		"traefik.enable": "true",
+		"traefik.http.routers.tunnel-king73-taylor-co-com-router.rule":                       "Host(`king73.taylor-co.com`)",
+		"traefik.http.routers.tunnel-king73-taylor-co-com-router.entrypoints":                "websecure",
+		"traefik.http.services.tunnel-king73-taylor-co-com-service.loadbalancer.server.port": "10007",
+		"traefik.http.middlewares.tunnel-king73-taylor-co-com-auth.basicauth.users":          "user:hash",
+	}
+
+	tunnelKeys, authKeys := collectExistingTunnelKeys(labels)
+
+	if !tunnelKeys["king73-taylor-co-com"] {
+		t.Fatalf("expected tunnel key to be adopted")
+	}
+	if !authKeys["king73-taylor-co-com"] {
+		t.Fatalf("expected auth key to be adopted")
+	}
+	if tunnelKeys["not-a-tunnel"] {
+		t.Fatalf("unexpected non-tunnel key")
+	}
+}
+
+func TestTunnelKeyFromLabelRejectsInvalidLabels(t *testing.T) {
+	tests := []string{
+		"traefik.enable",
+		"traefik.http.routers.tunnel--router.rule",
+		"traefik.http.routers.not-tunnel-king73-router.rule",
+	}
+
+	for _, test := range tests {
+		if key, ok := tunnelKeyFromLabel(test, "traefik.http.routers.tunnel-", "-router."); ok {
+			t.Fatalf("expected %q to be rejected, got key %q", test, key)
+		}
+	}
+}

internal/server/tunnel.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"hash/fnv"
 	"io"
 	"log"
 	"net"
@@ -46,6 +47,39 @@ func (pp *PortPool) Allocate() (int, error) {
 	return 0, fmt.Errorf("no ports available in range %d-%d", pp.start, pp.end)
 }
 
+// AllocatePreferred claims a stable port for key when possible.
+// If the preferred port is busy, it probes the range deterministically.
+func (pp *PortPool) AllocatePreferred(key string) (int, error) {
+	if strings.TrimSpace(key) == "" {
+		return pp.Allocate()
+	}
+
+	pp.mu.Lock()
+	defer pp.mu.Unlock()
+
+	span := pp.end - pp.start + 1
+	if span <= 0 {
+		return 0, fmt.Errorf("invalid port range %d-%d", pp.start, pp.end)
+	}
+
+	preferred := pp.start + stableOffset(key, span)
+	for i := 0; i < span; i++ {
+		port := pp.start + ((preferred - pp.start + i) % span)
+		if pp.available[port] {
+			pp.available[port] = false
+			return port, nil
+		}
+	}
+
+	return 0, fmt.Errorf("no ports available in range %d-%d", pp.start, pp.end)
+}
+
+func stableOffset(key string, span int) int {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(strings.ToLower(strings.TrimSpace(key))))
+	return int(h.Sum32() % uint32(span))
+}
+
 // Release returns a port to the pool.
 func (pp *PortPool) Release(port int) {
 	pp.mu.Lock()
@@ -105,8 +139,20 @@ func (s *SSHServer) handleForwardRequest(
 
 	log.Printf("tcpip-forward request: bind=%s:%d from %s", fwdReq.BindAddr, fwdReq.BindPort, connKey)
 
-	// Allocate a port from the pool.
+	// Determine the domain before allocating so reconnects reuse a stable port.
-	port, err := s.pool.Allocate()
+	// Look up the metadata channel first, fall back to bind address.
+	domain := fwdReq.BindAddr
+	var authUser, authPass string
+	s.mu.Lock()
+	if meta, ok := s.activeTuns[connKey+"-meta"]; ok {
+		domain = meta.domain
+		authUser = meta.authUser
+		authPass = meta.authPass
+	}
+	s.mu.Unlock()
+
+	// Allocate a stable port from the pool.
+	port, err := s.pool.AllocatePreferred(domain)
 	if err != nil {
 		log.Printf("port allocation failed: %v", err)
 		req.Reply(false, nil)
@@ -135,18 +181,6 @@ func (s *SSHServer) handleForwardRequest(
 		acceptForwardedConnections(listener, sshConn, fwdReq.BindAddr, uint32(port))
 	}()
 
-	// Determine the domain for Traefik label registration.
-	// Look up the metadata channel first, fall back to bind address.
-	domain := fwdReq.BindAddr
-	var authUser, authPass string
-	s.mu.Lock()
-	if meta, ok := s.activeTuns[connKey+"-meta"]; ok {
-		domain = meta.domain
-		authUser = meta.authUser
-		authPass = meta.authPass
-	}
-	s.mu.Unlock()
-
 	tunKey := SanitizeDomain(domain)
 	if tunKey == "" {
 		tunKey = fmt.Sprintf("port-%d", port)

internal/server/tunnel_test.go

@@ -0,0 +1,49 @@
+package server
+
+import "testing"
+
+func TestAllocatePreferredUsesStablePort(t *testing.T) {
+	first := NewPortPool(10000, 10010)
+	second := NewPortPool(10000, 10010)
+
+	firstPort, err := first.AllocatePreferred("king73.taylor-co.com")
+	if err != nil {
+		t.Fatalf("first allocate: %v", err)
+	}
+
+	secondPort, err := second.AllocatePreferred("king73.taylor-co.com")
+	if err != nil {
+		t.Fatalf("second allocate: %v", err)
+	}
+
+	if firstPort != secondPort {
+		t.Fatalf("expected stable port, got %d and %d", firstPort, secondPort)
+	}
+}
+
+func TestAllocatePreferredProbesOnCollision(t *testing.T) {
+	pool := NewPortPool(10000, 10001)
+
+	firstPort, err := pool.AllocatePreferred("king73.taylor-co.com")
+	if err != nil {
+		t.Fatalf("first allocate: %v", err)
+	}
+
+	secondPort, err := pool.AllocatePreferred("king73.taylor-co.com")
+	if err != nil {
+		t.Fatalf("second allocate: %v", err)
+	}
+
+	if firstPort == secondPort {
+		t.Fatalf("expected collision probe to choose a different port, got %d", secondPort)
+	}
+}
+
+func TestSanitizeDomain(t *testing.T) {
+	got := SanitizeDomain(" King73.Taylor-Co.com:443/path ")
+	want := "king73-taylor-co-com-443-path"
+
+	if got != want {
+		t.Fatalf("SanitizeDomain() = %q, want %q", got, want)
+	}
+}

systemd/tunnel-server.env.example

@@ -9,6 +9,9 @@ TRAEFIK_SSH_KEY=/etc/tunnel-server/traefik_deploy_key
 # AUTHORIZED_KEYS=/etc/tunnel-server/authorized_keys  (tunnel user's authorized_keys from ingress.nixc.us)
 # PORT_RANGE_START=10000
 # PORT_RANGE_END=10100
+# PURGE_STALE_LABELS_ON_START=false
+# HEALTHCHECK_ADDR=127.0.0.1:2222
+# HEALTHCHECK_TIMEOUT_SECONDS=2
 # TRAEFIK_SSH_USER=root
 # SWARM_SERVICE_NAME=better-argo-tunnels_tunnel-server
 # TRAEFIK_ENTRYPOINT=websecure