This commit is contained in:
Leopere 2026-04-26 15:58:23 -04:00
parent d8d18f6607
commit d35096afec
Signed by: colin
SSH Key Fingerprint: SHA256:nRPCQTeMFLdGytxRQmPVK9VXY3/ePKQ5lGRyJhT5DY8
8 changed files with 325 additions and 72 deletions

View File

@ -113,9 +113,11 @@ docker run -d \
| `TRAEFIK_SSH_HOST` | Traefik host to SSH into **(required)** | - |
| `TRAEFIK_SSH_USER` | SSH user on the Traefik host | `root` |
| `TRAEFIK_SSH_KEY` | SSH key for Traefik host (path or PEM) **(required)** | - |
| `TRAEFIK_CONFIG_DIR` | Remote path for dynamic configs | `/root/traefik/dynamic` |
| `TRAEFIK_ENTRYPOINT` | Traefik entrypoint name | `websecure` |
| `TRAEFIK_CERT_RESOLVER` | Traefik TLS cert resolver | `letsencryptresolver` |
| `PURGE_STALE_LABELS_ON_START` | Remove all managed tunnel labels during startup; normally leave disabled so routes survive restarts | `false` |
| `HEALTHCHECK_ADDR` | TCP address checked by `tunnel-server healthcheck` | `127.0.0.1:$SSH_PORT` |
| `HEALTHCHECK_TIMEOUT_SECONDS` | Healthcheck TCP dial timeout | `2` |
### Client
@ -247,6 +249,27 @@ If you run the tunnel server without Docker:
## Troubleshooting
### Self-healing after ingress restarts
The tunnel-server keeps Traefik tunnel labels across normal restarts and
reconciles them on startup instead of deleting every route. Tunnel ports are
chosen from a stable hash of the domain, so a reconnecting client gets the same
port and preserved labels point back to the right listener once the client
returns.
Use a non-interactive healthcheck to verify the SSH ingress port:
```bash
tunnel-server healthcheck
```
For Swarm deployments, configure the service healthcheck to run the same command
inside the container. The production stack example includes this healthcheck
without requiring a Dockerfile change.
If you intentionally need to clear every managed Traefik tunnel route, start the
server with `PURGE_STALE_LABELS_ON_START=true` once, then disable it again.
### "unable to authenticate, attempted methods [none publickey]"
This means the tunnel-server rejected the client's key. There are exactly two causes:
@ -263,24 +286,14 @@ ssh root@ingress.nixc.us "cat /home/tunnel/.ssh/authorized_keys"
ssh root@ingress.nixc.us "cat >> /home/tunnel/.ssh/authorized_keys" < YOUR_CLIENT_KEY.pub
```
**2. The tunnel-server was not restarted after adding the key.**
The tunnel-server loads `authorized_keys` into memory **once at startup**. Editing the file on disk does nothing until the container restarts. After adding a key, always restart:
```bash
ssh root@ingress.nixc.us "docker service update --force better-argo-tunnels_tunnel-server"
```
Then restart the client so it reconnects immediately instead of waiting for its backoff timer.
### Checklist for a new tunnel client
1. Generate an ed25519 key (or reuse an existing one).
2. Append the `.pub` to `/home/tunnel/.ssh/authorized_keys` on `ingress.nixc.us`.
3. Restart the tunnel-server: `docker service update --force better-argo-tunnels_tunnel-server`.
4. Start the client container with the private key mounted and `TUNNEL_KEY` pointing at it.
3. Start the client container with the private key mounted and `TUNNEL_KEY` pointing at it.
If all four steps happen, the tunnel will connect. If any step is skipped, it won't.
The tunnel-server reloads `authorized_keys` on each auth attempt, so new keys do
not require a server restart.
## Security Notes

View File

@ -33,6 +33,12 @@ services:
published: 2222
protocol: tcp
mode: host
healthcheck:
test: ["CMD", "tunnel-server", "healthcheck"]
interval: 10s
timeout: 3s
retries: 3
start_period: 10s
deploy:
replicas: 1
placement:

View File

@ -2,10 +2,12 @@ package main
import (
"log"
"net"
"os"
"os/signal"
"strconv"
"syscall"
"time"
"github.com/nixc/reverse-ssh-traefik/internal/buildinfo"
"github.com/nixc/reverse-ssh-traefik/internal/server"
@ -39,8 +41,27 @@ func envInt(key string, fallback int) int {
return n
}
func envBool(key string, fallback bool) bool {
v := os.Getenv(key)
if v == "" {
return fallback
}
b, err := strconv.ParseBool(v)
if err != nil {
log.Printf("WARN: invalid %s=%q, using default %t", key, v, fallback)
return fallback
}
return b
}
func main() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
if len(os.Args) > 1 && os.Args[1] == "healthcheck" {
runHealthcheck()
return
}
log.Printf("tunnel-server starting commit=%s", buildinfo.Commit)
// SSH server config (for accepting tunnel clients).
@ -79,9 +100,13 @@ func main() {
}
defer labels.Close()
// Purge stale tunnel labels left by a previous container instance.
if err := labels.PurgeAllTunnelLabels(); err != nil {
log.Printf("WARN: failed to purge stale labels: %v", err)
if envBool("PURGE_STALE_LABELS_ON_START", false) {
log.Printf("PURGE_STALE_LABELS_ON_START enabled; removing all tunnel labels")
if err := labels.PurgeAllTunnelLabels(); err != nil {
log.Printf("WARN: failed to purge stale labels: %v", err)
}
} else if err := labels.ReconcileExistingTunnelLabels(); err != nil {
log.Printf("WARN: failed to reconcile existing tunnel labels: %v", err)
}
// Initialize SSH server for tunnel clients.
@ -105,3 +130,16 @@ func main() {
log.Fatalf("SSH server error: %v", err)
}
}
func runHealthcheck() {
sshPort := envOr("SSH_PORT", "2222")
addr := envOr("HEALTHCHECK_ADDR", "127.0.0.1:"+sshPort)
timeout := time.Duration(envInt("HEALTHCHECK_TIMEOUT_SECONDS", 2)) * time.Second
conn, err := net.DialTimeout("tcp", addr, timeout)
if err != nil {
log.Printf("healthcheck failed for %s: %v", addr, err)
os.Exit(1)
}
conn.Close()
}

View File

@ -18,9 +18,9 @@ type LabelManager struct {
remoteHost string // Swarm manager, e.g. "ingress.nixc.us"
remoteUser string // SSH user
signer ssh.Signer
serviceName string // Swarm service name, e.g. "better-argo-tunnels_tunnel-server"
entrypoint string // e.g. "websecure"
certResolver string // e.g. "letsencryptresolver"
serviceName string // Swarm service name, e.g. "better-argo-tunnels_tunnel-server"
entrypoint string // e.g. "websecure"
certResolver string // e.g. "letsencryptresolver"
labels map[string]bool // track which tunnel keys we've added
authLabels map[string]bool // track which tunnel keys have auth middleware
}
@ -57,50 +57,42 @@ func NewLabelManager(
return lm, nil
}
// ReconcileExistingTunnelLabels adopts existing tunnel-* Traefik labels from
// the Swarm service so a restarted server preserves routes while clients
// reconnect. It does not remove labels; explicit purge remains opt-in.
func (lm *LabelManager) ReconcileExistingTunnelLabels() error {
labels, err := lm.inspectLabels()
if err != nil {
return err
}
tunnelKeys, authKeys := collectExistingTunnelKeys(labels)
lm.mu.Lock()
for key := range tunnelKeys {
lm.labels[key] = true
}
for key := range authKeys {
lm.authLabels[key] = true
}
lm.mu.Unlock()
log.Printf("Reconciled %d existing tunnel route(s), %d with auth", len(tunnelKeys), len(authKeys))
return nil
}
// PurgeAllTunnelLabels removes every tunnel-* Traefik label from the Swarm
// service. Called on startup so stale labels from a previous instance don't
// route traffic to ports that no longer exist.
// service. This is intentionally opt-in because purging on every restart
// creates a 404 window until every client reconnects.
func (lm *LabelManager) PurgeAllTunnelLabels() error {
addr := lm.remoteHost
if !strings.Contains(addr, ":") {
addr = addr + ":22"
}
config := &ssh.ClientConfig{
User: lm.remoteUser,
Auth: []ssh.AuthMethod{ssh.PublicKeys(lm.signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
client, err := ssh.Dial("tcp", addr, config)
labels, err := lm.inspectLabels()
if err != nil {
return fmt.Errorf("SSH dial %s: %w", addr, err)
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return fmt.Errorf("SSH session: %w", err)
}
defer session.Close()
inspectCmd := fmt.Sprintf(
"docker service inspect --format '{{json .Spec.Labels}}' %s", lm.serviceName)
output, err := session.CombinedOutput(inspectCmd)
if err != nil {
return fmt.Errorf("inspect labels: %w (%s)", err, string(output))
}
var labels map[string]string
if err := json.Unmarshal(output, &labels); err != nil {
return fmt.Errorf("parse labels JSON: %w", err)
return err
}
var rmLabels []string
for key := range labels {
if strings.HasPrefix(key, "traefik.http.routers.tunnel-") ||
strings.HasPrefix(key, "traefik.http.services.tunnel-") ||
strings.HasPrefix(key, "traefik.http.middlewares.tunnel-") {
if isTunnelLabel(key) {
rmLabels = append(rmLabels, key)
}
}
@ -116,10 +108,89 @@ func (lm *LabelManager) PurgeAllTunnelLabels() error {
return fmt.Errorf("purge tunnel labels: %w", err)
}
log.Printf("Purged %d stale tunnel labels on startup", len(rmLabels))
log.Printf("Purged %d tunnel label(s)", len(rmLabels))
return nil
}
func (lm *LabelManager) inspectLabels() (map[string]string, error) {
addr := lm.remoteHost
if !strings.Contains(addr, ":") {
addr = addr + ":22"
}
config := &ssh.ClientConfig{
User: lm.remoteUser,
Auth: []ssh.AuthMethod{ssh.PublicKeys(lm.signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
client, err := ssh.Dial("tcp", addr, config)
if err != nil {
return nil, fmt.Errorf("SSH dial %s: %w", addr, err)
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return nil, fmt.Errorf("SSH session: %w", err)
}
defer session.Close()
inspectCmd := fmt.Sprintf(
"docker service inspect --format '{{json .Spec.Labels}}' %s", lm.serviceName)
output, err := session.CombinedOutput(inspectCmd)
if err != nil {
return nil, fmt.Errorf("inspect labels: %w (%s)", err, string(output))
}
var labels map[string]string
if err := json.Unmarshal(output, &labels); err != nil {
return nil, fmt.Errorf("parse labels JSON: %w", err)
}
return labels, nil
}
func isTunnelLabel(key string) bool {
return strings.HasPrefix(key, "traefik.http.routers.tunnel-") ||
strings.HasPrefix(key, "traefik.http.services.tunnel-") ||
strings.HasPrefix(key, "traefik.http.middlewares.tunnel-")
}
func collectExistingTunnelKeys(labels map[string]string) (map[string]bool, map[string]bool) {
tunnelKeys := make(map[string]bool)
authKeys := make(map[string]bool)
for label := range labels {
if key, ok := tunnelKeyFromLabel(label, "traefik.http.routers.tunnel-", "-router."); ok {
tunnelKeys[key] = true
continue
}
if key, ok := tunnelKeyFromLabel(label, "traefik.http.services.tunnel-", "-service."); ok {
tunnelKeys[key] = true
continue
}
if key, ok := tunnelKeyFromLabel(label, "traefik.http.middlewares.tunnel-", "-auth."); ok {
tunnelKeys[key] = true
authKeys[key] = true
}
}
return tunnelKeys, authKeys
}
func tunnelKeyFromLabel(label, prefix, suffix string) (string, bool) {
if !strings.HasPrefix(label, prefix) {
return "", false
}
rest := strings.TrimPrefix(label, prefix)
idx := strings.Index(rest, suffix)
if idx <= 0 {
return "", false
}
return rest[:idx], true
}
// Add adds Traefik routing labels to the Swarm service for a tunnel.
// If authUser and authPass are non-empty, a basicauth middleware is also added.
func (lm *LabelManager) Add(tunKey, domain string, port int, authUser, authPass string) error {

View File

@ -0,0 +1,39 @@
package server
import "testing"
func TestCollectExistingTunnelKeys(t *testing.T) {
labels := map[string]string{
"traefik.enable": "true",
"traefik.http.routers.tunnel-king73-taylor-co-com-router.rule": "Host(`king73.taylor-co.com`)",
"traefik.http.routers.tunnel-king73-taylor-co-com-router.entrypoints": "websecure",
"traefik.http.services.tunnel-king73-taylor-co-com-service.loadbalancer.server.port": "10007",
"traefik.http.middlewares.tunnel-king73-taylor-co-com-auth.basicauth.users": "user:hash",
}
tunnelKeys, authKeys := collectExistingTunnelKeys(labels)
if !tunnelKeys["king73-taylor-co-com"] {
t.Fatalf("expected tunnel key to be adopted")
}
if !authKeys["king73-taylor-co-com"] {
t.Fatalf("expected auth key to be adopted")
}
if tunnelKeys["not-a-tunnel"] {
t.Fatalf("unexpected non-tunnel key")
}
}
func TestTunnelKeyFromLabelRejectsInvalidLabels(t *testing.T) {
tests := []string{
"traefik.enable",
"traefik.http.routers.tunnel--router.rule",
"traefik.http.routers.not-tunnel-king73-router.rule",
}
for _, test := range tests {
if key, ok := tunnelKeyFromLabel(test, "traefik.http.routers.tunnel-", "-router."); ok {
t.Fatalf("expected %q to be rejected, got key %q", test, key)
}
}
}

View File

@ -2,6 +2,7 @@ package server
import (
"fmt"
"hash/fnv"
"io"
"log"
"net"
@ -46,6 +47,39 @@ func (pp *PortPool) Allocate() (int, error) {
return 0, fmt.Errorf("no ports available in range %d-%d", pp.start, pp.end)
}
// AllocatePreferred claims a stable port for key when possible.
// If the preferred port is busy, it probes the range deterministically.
func (pp *PortPool) AllocatePreferred(key string) (int, error) {
if strings.TrimSpace(key) == "" {
return pp.Allocate()
}
pp.mu.Lock()
defer pp.mu.Unlock()
span := pp.end - pp.start + 1
if span <= 0 {
return 0, fmt.Errorf("invalid port range %d-%d", pp.start, pp.end)
}
preferred := pp.start + stableOffset(key, span)
for i := 0; i < span; i++ {
port := pp.start + ((preferred - pp.start + i) % span)
if pp.available[port] {
pp.available[port] = false
return port, nil
}
}
return 0, fmt.Errorf("no ports available in range %d-%d", pp.start, pp.end)
}
func stableOffset(key string, span int) int {
h := fnv.New32a()
_, _ = h.Write([]byte(strings.ToLower(strings.TrimSpace(key))))
return int(h.Sum32() % uint32(span))
}
// Release returns a port to the pool.
func (pp *PortPool) Release(port int) {
pp.mu.Lock()
@ -105,8 +139,20 @@ func (s *SSHServer) handleForwardRequest(
log.Printf("tcpip-forward request: bind=%s:%d from %s", fwdReq.BindAddr, fwdReq.BindPort, connKey)
// Allocate a port from the pool.
port, err := s.pool.Allocate()
// Determine the domain before allocating so reconnects reuse a stable port.
// Look up the metadata channel first, fall back to bind address.
domain := fwdReq.BindAddr
var authUser, authPass string
s.mu.Lock()
if meta, ok := s.activeTuns[connKey+"-meta"]; ok {
domain = meta.domain
authUser = meta.authUser
authPass = meta.authPass
}
s.mu.Unlock()
// Allocate a stable port from the pool.
port, err := s.pool.AllocatePreferred(domain)
if err != nil {
log.Printf("port allocation failed: %v", err)
req.Reply(false, nil)
@ -135,18 +181,6 @@ func (s *SSHServer) handleForwardRequest(
acceptForwardedConnections(listener, sshConn, fwdReq.BindAddr, uint32(port))
}()
// Determine the domain for Traefik label registration.
// Look up the metadata channel first, fall back to bind address.
domain := fwdReq.BindAddr
var authUser, authPass string
s.mu.Lock()
if meta, ok := s.activeTuns[connKey+"-meta"]; ok {
domain = meta.domain
authUser = meta.authUser
authPass = meta.authPass
}
s.mu.Unlock()
tunKey := SanitizeDomain(domain)
if tunKey == "" {
tunKey = fmt.Sprintf("port-%d", port)

View File

@ -0,0 +1,49 @@
package server
import "testing"
func TestAllocatePreferredUsesStablePort(t *testing.T) {
first := NewPortPool(10000, 10010)
second := NewPortPool(10000, 10010)
firstPort, err := first.AllocatePreferred("king73.taylor-co.com")
if err != nil {
t.Fatalf("first allocate: %v", err)
}
secondPort, err := second.AllocatePreferred("king73.taylor-co.com")
if err != nil {
t.Fatalf("second allocate: %v", err)
}
if firstPort != secondPort {
t.Fatalf("expected stable port, got %d and %d", firstPort, secondPort)
}
}
func TestAllocatePreferredProbesOnCollision(t *testing.T) {
pool := NewPortPool(10000, 10001)
firstPort, err := pool.AllocatePreferred("king73.taylor-co.com")
if err != nil {
t.Fatalf("first allocate: %v", err)
}
secondPort, err := pool.AllocatePreferred("king73.taylor-co.com")
if err != nil {
t.Fatalf("second allocate: %v", err)
}
if firstPort == secondPort {
t.Fatalf("expected collision probe to choose a different port, got %d", secondPort)
}
}
func TestSanitizeDomain(t *testing.T) {
got := SanitizeDomain(" King73.Taylor-Co.com:443/path ")
want := "king73-taylor-co-com-443-path"
if got != want {
t.Fatalf("SanitizeDomain() = %q, want %q", got, want)
}
}

View File

@ -9,6 +9,9 @@ TRAEFIK_SSH_KEY=/etc/tunnel-server/traefik_deploy_key
# AUTHORIZED_KEYS=/etc/tunnel-server/authorized_keys (tunnel user's authorized_keys from ingress.nixc.us)
# PORT_RANGE_START=10000
# PORT_RANGE_END=10100
# PURGE_STALE_LABELS_ON_START=false
# HEALTHCHECK_ADDR=127.0.0.1:2222
# HEALTHCHECK_TIMEOUT_SECONDS=2
# TRAEFIK_SSH_USER=root
# SWARM_SERVICE_NAME=better-argo-tunnels_tunnel-server
# TRAEFIK_ENTRYPOINT=websecure