diff --git a/README.md b/README.md index 707488b..8a86b98 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,11 @@ docker run -d \ | `TRAEFIK_SSH_HOST` | Traefik host to SSH into **(required)** | - | | `TRAEFIK_SSH_USER` | SSH user on the Traefik host | `root` | | `TRAEFIK_SSH_KEY` | SSH key for Traefik host (path or PEM) **(required)** | - | -| `TRAEFIK_CONFIG_DIR` | Remote path for dynamic configs | `/root/traefik/dynamic` | | `TRAEFIK_ENTRYPOINT` | Traefik entrypoint name | `websecure` | | `TRAEFIK_CERT_RESOLVER` | Traefik TLS cert resolver | `letsencryptresolver` | +| `PURGE_STALE_LABELS_ON_START` | Remove all managed tunnel labels during startup; normally leave disabled so routes survive restarts | `false` | +| `HEALTHCHECK_ADDR` | TCP address checked by `tunnel-server healthcheck` | `127.0.0.1:$SSH_PORT` | +| `HEALTHCHECK_TIMEOUT_SECONDS` | Healthcheck TCP dial timeout | `2` | ### Client @@ -247,6 +249,27 @@ If you run the tunnel server without Docker: ## Troubleshooting +### Self-healing after ingress restarts + +The tunnel-server keeps Traefik tunnel labels across normal restarts and +reconciles them on startup instead of deleting every route. Tunnel ports are +chosen from a stable hash of the domain, so a reconnecting client gets the same +port and preserved labels point back to the right listener once the client +returns. + +Use a non-interactive healthcheck to verify the SSH ingress port: + +```bash +tunnel-server healthcheck +``` + +For Swarm deployments, configure the service healthcheck to run the same command +inside the container. The production stack example includes this healthcheck +without requiring a Dockerfile change. + +If you intentionally need to clear every managed Traefik tunnel route, start the +server with `PURGE_STALE_LABELS_ON_START=true` once, then disable it again. + ### "unable to authenticate, attempted methods [none publickey]" This means the tunnel-server rejected the client's key. There are exactly two causes: @@ -263,24 +286,14 @@ ssh root@ingress.nixc.us "cat /home/tunnel/.ssh/authorized_keys" ssh root@ingress.nixc.us "cat >> /home/tunnel/.ssh/authorized_keys" < YOUR_CLIENT_KEY.pub ``` -**2. The tunnel-server was not restarted after adding the key.** - -The tunnel-server loads `authorized_keys` into memory **once at startup**. Editing the file on disk does nothing until the container restarts. After adding a key, always restart: - -```bash -ssh root@ingress.nixc.us "docker service update --force better-argo-tunnels_tunnel-server" -``` - -Then restart the client so it reconnects immediately instead of waiting for its backoff timer. - ### Checklist for a new tunnel client 1. Generate an ed25519 key (or reuse an existing one). 2. Append the `.pub` to `/home/tunnel/.ssh/authorized_keys` on `ingress.nixc.us`. -3. Restart the tunnel-server: `docker service update --force better-argo-tunnels_tunnel-server`. -4. Start the client container with the private key mounted and `TUNNEL_KEY` pointing at it. +3. Start the client container with the private key mounted and `TUNNEL_KEY` pointing at it. -If all four steps happen, the tunnel will connect. If any step is skipped, it won't. +The tunnel-server reloads `authorized_keys` on each auth attempt, so new keys do +not require a server restart. ## Security Notes diff --git a/archive/stack.production.yml b/archive/stack.production.yml index 77c2da6..8a338ad 100644 --- a/archive/stack.production.yml +++ b/archive/stack.production.yml @@ -33,6 +33,12 @@ services: published: 2222 protocol: tcp mode: host + healthcheck: + test: ["CMD", "tunnel-server", "healthcheck"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 10s deploy: replicas: 1 placement: diff --git a/cmd/server/main.go b/cmd/server/main.go index 3c1612f..8e49468 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,10 +2,12 @@ package main import ( "log" + "net" "os" "os/signal" "strconv" "syscall" + "time" "github.com/nixc/reverse-ssh-traefik/internal/buildinfo" "github.com/nixc/reverse-ssh-traefik/internal/server" @@ -39,8 +41,27 @@ func envInt(key string, fallback int) int { return n } +func envBool(key string, fallback bool) bool { + v := os.Getenv(key) + if v == "" { + return fallback + } + b, err := strconv.ParseBool(v) + if err != nil { + log.Printf("WARN: invalid %s=%q, using default %t", key, v, fallback) + return fallback + } + return b +} + func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) + + if len(os.Args) > 1 && os.Args[1] == "healthcheck" { + runHealthcheck() + return + } + log.Printf("tunnel-server starting commit=%s", buildinfo.Commit) // SSH server config (for accepting tunnel clients). @@ -79,9 +100,13 @@ func main() { } defer labels.Close() - // Purge stale tunnel labels left by a previous container instance. - if err := labels.PurgeAllTunnelLabels(); err != nil { - log.Printf("WARN: failed to purge stale labels: %v", err) + if envBool("PURGE_STALE_LABELS_ON_START", false) { + log.Printf("PURGE_STALE_LABELS_ON_START enabled; removing all tunnel labels") + if err := labels.PurgeAllTunnelLabels(); err != nil { + log.Printf("WARN: failed to purge stale labels: %v", err) + } + } else if err := labels.ReconcileExistingTunnelLabels(); err != nil { + log.Printf("WARN: failed to reconcile existing tunnel labels: %v", err) } // Initialize SSH server for tunnel clients. @@ -105,3 +130,16 @@ func main() { log.Fatalf("SSH server error: %v", err) } } + +func runHealthcheck() { + sshPort := envOr("SSH_PORT", "2222") + addr := envOr("HEALTHCHECK_ADDR", "127.0.0.1:"+sshPort) + timeout := time.Duration(envInt("HEALTHCHECK_TIMEOUT_SECONDS", 2)) * time.Second + + conn, err := net.DialTimeout("tcp", addr, timeout) + if err != nil { + log.Printf("healthcheck failed for %s: %v", addr, err) + os.Exit(1) + } + conn.Close() +} diff --git a/internal/server/labels.go b/internal/server/labels.go index f4353f5..92920ef 100644 --- a/internal/server/labels.go +++ b/internal/server/labels.go @@ -18,9 +18,9 @@ type LabelManager struct { remoteHost string // Swarm manager, e.g. "ingress.nixc.us" remoteUser string // SSH user signer ssh.Signer - serviceName string // Swarm service name, e.g. "better-argo-tunnels_tunnel-server" - entrypoint string // e.g. "websecure" - certResolver string // e.g. "letsencryptresolver" + serviceName string // Swarm service name, e.g. "better-argo-tunnels_tunnel-server" + entrypoint string // e.g. "websecure" + certResolver string // e.g. "letsencryptresolver" labels map[string]bool // track which tunnel keys we've added authLabels map[string]bool // track which tunnel keys have auth middleware } @@ -57,50 +57,42 @@ func NewLabelManager( return lm, nil } +// ReconcileExistingTunnelLabels adopts existing tunnel-* Traefik labels from +// the Swarm service so a restarted server preserves routes while clients +// reconnect. It does not remove labels; explicit purge remains opt-in. +func (lm *LabelManager) ReconcileExistingTunnelLabels() error { + labels, err := lm.inspectLabels() + if err != nil { + return err + } + + tunnelKeys, authKeys := collectExistingTunnelKeys(labels) + + lm.mu.Lock() + for key := range tunnelKeys { + lm.labels[key] = true + } + for key := range authKeys { + lm.authLabels[key] = true + } + lm.mu.Unlock() + + log.Printf("Reconciled %d existing tunnel route(s), %d with auth", len(tunnelKeys), len(authKeys)) + return nil +} + // PurgeAllTunnelLabels removes every tunnel-* Traefik label from the Swarm -// service. Called on startup so stale labels from a previous instance don't -// route traffic to ports that no longer exist. +// service. This is intentionally opt-in because purging on every restart +// creates a 404 window until every client reconnects. func (lm *LabelManager) PurgeAllTunnelLabels() error { - addr := lm.remoteHost - if !strings.Contains(addr, ":") { - addr = addr + ":22" - } - - config := &ssh.ClientConfig{ - User: lm.remoteUser, - Auth: []ssh.AuthMethod{ssh.PublicKeys(lm.signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - client, err := ssh.Dial("tcp", addr, config) + labels, err := lm.inspectLabels() if err != nil { - return fmt.Errorf("SSH dial %s: %w", addr, err) - } - defer client.Close() - - session, err := client.NewSession() - if err != nil { - return fmt.Errorf("SSH session: %w", err) - } - defer session.Close() - - inspectCmd := fmt.Sprintf( - "docker service inspect --format '{{json .Spec.Labels}}' %s", lm.serviceName) - output, err := session.CombinedOutput(inspectCmd) - if err != nil { - return fmt.Errorf("inspect labels: %w (%s)", err, string(output)) - } - - var labels map[string]string - if err := json.Unmarshal(output, &labels); err != nil { - return fmt.Errorf("parse labels JSON: %w", err) + return err } var rmLabels []string for key := range labels { - if strings.HasPrefix(key, "traefik.http.routers.tunnel-") || - strings.HasPrefix(key, "traefik.http.services.tunnel-") || - strings.HasPrefix(key, "traefik.http.middlewares.tunnel-") { + if isTunnelLabel(key) { rmLabels = append(rmLabels, key) } } @@ -116,10 +108,89 @@ func (lm *LabelManager) PurgeAllTunnelLabels() error { return fmt.Errorf("purge tunnel labels: %w", err) } - log.Printf("Purged %d stale tunnel labels on startup", len(rmLabels)) + log.Printf("Purged %d tunnel label(s)", len(rmLabels)) return nil } +func (lm *LabelManager) inspectLabels() (map[string]string, error) { + addr := lm.remoteHost + if !strings.Contains(addr, ":") { + addr = addr + ":22" + } + + config := &ssh.ClientConfig{ + User: lm.remoteUser, + Auth: []ssh.AuthMethod{ssh.PublicKeys(lm.signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + client, err := ssh.Dial("tcp", addr, config) + if err != nil { + return nil, fmt.Errorf("SSH dial %s: %w", addr, err) + } + defer client.Close() + + session, err := client.NewSession() + if err != nil { + return nil, fmt.Errorf("SSH session: %w", err) + } + defer session.Close() + + inspectCmd := fmt.Sprintf( + "docker service inspect --format '{{json .Spec.Labels}}' %s", lm.serviceName) + output, err := session.CombinedOutput(inspectCmd) + if err != nil { + return nil, fmt.Errorf("inspect labels: %w (%s)", err, string(output)) + } + + var labels map[string]string + if err := json.Unmarshal(output, &labels); err != nil { + return nil, fmt.Errorf("parse labels JSON: %w", err) + } + + return labels, nil +} + +func isTunnelLabel(key string) bool { + return strings.HasPrefix(key, "traefik.http.routers.tunnel-") || + strings.HasPrefix(key, "traefik.http.services.tunnel-") || + strings.HasPrefix(key, "traefik.http.middlewares.tunnel-") +} + +func collectExistingTunnelKeys(labels map[string]string) (map[string]bool, map[string]bool) { + tunnelKeys := make(map[string]bool) + authKeys := make(map[string]bool) + + for label := range labels { + if key, ok := tunnelKeyFromLabel(label, "traefik.http.routers.tunnel-", "-router."); ok { + tunnelKeys[key] = true + continue + } + if key, ok := tunnelKeyFromLabel(label, "traefik.http.services.tunnel-", "-service."); ok { + tunnelKeys[key] = true + continue + } + if key, ok := tunnelKeyFromLabel(label, "traefik.http.middlewares.tunnel-", "-auth."); ok { + tunnelKeys[key] = true + authKeys[key] = true + } + } + + return tunnelKeys, authKeys +} + +func tunnelKeyFromLabel(label, prefix, suffix string) (string, bool) { + if !strings.HasPrefix(label, prefix) { + return "", false + } + rest := strings.TrimPrefix(label, prefix) + idx := strings.Index(rest, suffix) + if idx <= 0 { + return "", false + } + return rest[:idx], true +} + // Add adds Traefik routing labels to the Swarm service for a tunnel. // If authUser and authPass are non-empty, a basicauth middleware is also added. func (lm *LabelManager) Add(tunKey, domain string, port int, authUser, authPass string) error { diff --git a/internal/server/labels_test.go b/internal/server/labels_test.go new file mode 100644 index 0000000..ba0e0cf --- /dev/null +++ b/internal/server/labels_test.go @@ -0,0 +1,39 @@ +package server + +import "testing" + +func TestCollectExistingTunnelKeys(t *testing.T) { + labels := map[string]string{ + "traefik.enable": "true", + "traefik.http.routers.tunnel-king73-taylor-co-com-router.rule": "Host(`king73.taylor-co.com`)", + "traefik.http.routers.tunnel-king73-taylor-co-com-router.entrypoints": "websecure", + "traefik.http.services.tunnel-king73-taylor-co-com-service.loadbalancer.server.port": "10007", + "traefik.http.middlewares.tunnel-king73-taylor-co-com-auth.basicauth.users": "user:hash", + } + + tunnelKeys, authKeys := collectExistingTunnelKeys(labels) + + if !tunnelKeys["king73-taylor-co-com"] { + t.Fatalf("expected tunnel key to be adopted") + } + if !authKeys["king73-taylor-co-com"] { + t.Fatalf("expected auth key to be adopted") + } + if tunnelKeys["not-a-tunnel"] { + t.Fatalf("unexpected non-tunnel key") + } +} + +func TestTunnelKeyFromLabelRejectsInvalidLabels(t *testing.T) { + tests := []string{ + "traefik.enable", + "traefik.http.routers.tunnel--router.rule", + "traefik.http.routers.not-tunnel-king73-router.rule", + } + + for _, test := range tests { + if key, ok := tunnelKeyFromLabel(test, "traefik.http.routers.tunnel-", "-router."); ok { + t.Fatalf("expected %q to be rejected, got key %q", test, key) + } + } +} diff --git a/internal/server/tunnel.go b/internal/server/tunnel.go index 0a135b8..59ee2f3 100644 --- a/internal/server/tunnel.go +++ b/internal/server/tunnel.go @@ -2,6 +2,7 @@ package server import ( "fmt" + "hash/fnv" "io" "log" "net" @@ -46,6 +47,39 @@ func (pp *PortPool) Allocate() (int, error) { return 0, fmt.Errorf("no ports available in range %d-%d", pp.start, pp.end) } +// AllocatePreferred claims a stable port for key when possible. +// If the preferred port is busy, it probes the range deterministically. +func (pp *PortPool) AllocatePreferred(key string) (int, error) { + if strings.TrimSpace(key) == "" { + return pp.Allocate() + } + + pp.mu.Lock() + defer pp.mu.Unlock() + + span := pp.end - pp.start + 1 + if span <= 0 { + return 0, fmt.Errorf("invalid port range %d-%d", pp.start, pp.end) + } + + preferred := pp.start + stableOffset(key, span) + for i := 0; i < span; i++ { + port := pp.start + ((preferred - pp.start + i) % span) + if pp.available[port] { + pp.available[port] = false + return port, nil + } + } + + return 0, fmt.Errorf("no ports available in range %d-%d", pp.start, pp.end) +} + +func stableOffset(key string, span int) int { + h := fnv.New32a() + _, _ = h.Write([]byte(strings.ToLower(strings.TrimSpace(key)))) + return int(h.Sum32() % uint32(span)) +} + // Release returns a port to the pool. func (pp *PortPool) Release(port int) { pp.mu.Lock() @@ -105,8 +139,20 @@ func (s *SSHServer) handleForwardRequest( log.Printf("tcpip-forward request: bind=%s:%d from %s", fwdReq.BindAddr, fwdReq.BindPort, connKey) - // Allocate a port from the pool. - port, err := s.pool.Allocate() + // Determine the domain before allocating so reconnects reuse a stable port. + // Look up the metadata channel first, fall back to bind address. + domain := fwdReq.BindAddr + var authUser, authPass string + s.mu.Lock() + if meta, ok := s.activeTuns[connKey+"-meta"]; ok { + domain = meta.domain + authUser = meta.authUser + authPass = meta.authPass + } + s.mu.Unlock() + + // Allocate a stable port from the pool. + port, err := s.pool.AllocatePreferred(domain) if err != nil { log.Printf("port allocation failed: %v", err) req.Reply(false, nil) @@ -135,18 +181,6 @@ func (s *SSHServer) handleForwardRequest( acceptForwardedConnections(listener, sshConn, fwdReq.BindAddr, uint32(port)) }() - // Determine the domain for Traefik label registration. - // Look up the metadata channel first, fall back to bind address. - domain := fwdReq.BindAddr - var authUser, authPass string - s.mu.Lock() - if meta, ok := s.activeTuns[connKey+"-meta"]; ok { - domain = meta.domain - authUser = meta.authUser - authPass = meta.authPass - } - s.mu.Unlock() - tunKey := SanitizeDomain(domain) if tunKey == "" { tunKey = fmt.Sprintf("port-%d", port) diff --git a/internal/server/tunnel_test.go b/internal/server/tunnel_test.go new file mode 100644 index 0000000..9c82aab --- /dev/null +++ b/internal/server/tunnel_test.go @@ -0,0 +1,49 @@ +package server + +import "testing" + +func TestAllocatePreferredUsesStablePort(t *testing.T) { + first := NewPortPool(10000, 10010) + second := NewPortPool(10000, 10010) + + firstPort, err := first.AllocatePreferred("king73.taylor-co.com") + if err != nil { + t.Fatalf("first allocate: %v", err) + } + + secondPort, err := second.AllocatePreferred("king73.taylor-co.com") + if err != nil { + t.Fatalf("second allocate: %v", err) + } + + if firstPort != secondPort { + t.Fatalf("expected stable port, got %d and %d", firstPort, secondPort) + } +} + +func TestAllocatePreferredProbesOnCollision(t *testing.T) { + pool := NewPortPool(10000, 10001) + + firstPort, err := pool.AllocatePreferred("king73.taylor-co.com") + if err != nil { + t.Fatalf("first allocate: %v", err) + } + + secondPort, err := pool.AllocatePreferred("king73.taylor-co.com") + if err != nil { + t.Fatalf("second allocate: %v", err) + } + + if firstPort == secondPort { + t.Fatalf("expected collision probe to choose a different port, got %d", secondPort) + } +} + +func TestSanitizeDomain(t *testing.T) { + got := SanitizeDomain(" King73.Taylor-Co.com:443/path ") + want := "king73-taylor-co-com-443-path" + + if got != want { + t.Fatalf("SanitizeDomain() = %q, want %q", got, want) + } +} diff --git a/systemd/tunnel-server.env.example b/systemd/tunnel-server.env.example index 3dc68c1..5d3c789 100644 --- a/systemd/tunnel-server.env.example +++ b/systemd/tunnel-server.env.example @@ -9,6 +9,9 @@ TRAEFIK_SSH_KEY=/etc/tunnel-server/traefik_deploy_key # AUTHORIZED_KEYS=/etc/tunnel-server/authorized_keys (tunnel user's authorized_keys from ingress.nixc.us) # PORT_RANGE_START=10000 # PORT_RANGE_END=10100 +# PURGE_STALE_LABELS_ON_START=false +# HEALTHCHECK_ADDR=127.0.0.1:2222 +# HEALTHCHECK_TIMEOUT_SECONDS=2 # TRAEFIK_SSH_USER=root # SWARM_SERVICE_NAME=better-argo-tunnels_tunnel-server # TRAEFIK_ENTRYPOINT=websecure