feat(healthcheck): combination of ICMP and TCP+TLS checks (#2923)

- New option: `HEALTH_ICMP_TARGET_IP` defaults to `0.0.0.0` meaning use the VPN server public IP address.
- Options removed: `HEALTH_VPN_INITIAL_DURATION` and `HEALTH_VPN_ADDITIONAL_DURATION` - times and retries are handpicked and hardcoded.
- Less aggressive checks and less false positive detection

Author: qdm12

Dockerfile

@@ -164,9 +164,7 @@ ENV VPN_SERVICE_PROVIDER=pia \
     # Health
     HEALTH_SERVER_ADDRESS=127.0.0.1:9999 \
     HEALTH_TARGET_ADDRESS=cloudflare.com:443 \
-    HEALTH_SUCCESS_WAIT_DURATION=5s \
-    HEALTH_VPN_DURATION_INITIAL=6s \
-    HEALTH_VPN_DURATION_ADDITION=5s \
+    HEALTH_ICMP_TARGET_IP=0.0.0.0 \
     # DNS over TLS
     DOT=on \
     DOT_PROVIDERS=cloudflare \

cmd/gluetun/main.go

@@ -414,6 +414,13 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 		return fmt.Errorf("starting public ip loop: %w", err)
 	}
 
+	healthLogger := logger.New(log.SetComponent("healthcheck"))
+	healthcheckServer := healthcheck.NewServer(allSettings.Health, healthLogger)
+	healthServerHandler, healthServerCtx, healthServerDone := goshutdown.NewGoRoutineHandler(
+		"HTTP health server", goroutine.OptionTimeout(defaultShutdownTimeout))
+	go healthcheckServer.Run(healthServerCtx, healthServerDone)
+	healthChecker := healthcheck.NewChecker(healthLogger)
+
 	updaterLogger := logger.New(log.SetComponent("updater"))
 
 	unzipper := unzip.New(httpClient)
@@ -424,8 +431,8 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 
 	vpnLogger := logger.New(log.SetComponent("vpn"))
 	vpnLooper := vpn.NewLoop(allSettings.VPN, ipv6Supported, allSettings.Firewall.VPNInputPorts,
-		providers, storage, ovpnConf, netLinker, firewallConf, routingConf, portForwardLooper,
-		cmder, publicIPLooper, dnsLooper, vpnLogger, httpClient,
+		providers, storage, allSettings.Health, healthChecker, healthcheckServer, ovpnConf, netLinker, firewallConf,
+		routingConf, portForwardLooper, cmder, publicIPLooper, dnsLooper, vpnLogger, httpClient,
 		buildInfo, *allSettings.Version.Enabled)
 	vpnHandler, vpnCtx, vpnDone := goshutdown.NewGoRoutineHandler(
 		"vpn", goroutine.OptionTimeout(time.Second))
@@ -476,12 +483,6 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 	<-httpServerReady
 	controlGroupHandler.Add(httpServerHandler)
 
-	healthLogger := logger.New(log.SetComponent("healthcheck"))
-	healthcheckServer := healthcheck.NewServer(allSettings.Health, healthLogger, vpnLooper)
-	healthServerHandler, healthServerCtx, healthServerDone := goshutdown.NewGoRoutineHandler(
-		"HTTP health server", goroutine.OptionTimeout(defaultShutdownTimeout))
-	go healthcheckServer.Run(healthServerCtx, healthServerDone)
-
 	orderHandler := goshutdown.NewOrderHandler("gluetun",
 		order.OptionTimeout(totalShutdownTimeout),
 		order.OptionOnSuccess(defaultShutdownOnSuccess),

internal/configuration/settings/deprecated.go

@@ -9,9 +9,11 @@ import (
 
 func readObsolete(r *reader.Reader) (warnings []string) {
 	keyToMessage := map[string]string{
-		"DOT_VERBOSITY":           "DOT_VERBOSITY is obsolete, use LOG_LEVEL instead.",
-		"DOT_VERBOSITY_DETAILS":   "DOT_VERBOSITY_DETAILS is obsolete because it was specific to Unbound.",
-		"DOT_VALIDATION_LOGLEVEL": "DOT_VALIDATION_LOGLEVEL is obsolete because DNSSEC validation is not implemented.",
+		"DOT_VERBOSITY":                "DOT_VERBOSITY is obsolete, use LOG_LEVEL instead.",
+		"DOT_VERBOSITY_DETAILS":        "DOT_VERBOSITY_DETAILS is obsolete because it was specific to Unbound.",
+		"DOT_VALIDATION_LOGLEVEL":      "DOT_VALIDATION_LOGLEVEL is obsolete because DNSSEC validation is not implemented.",
+		"HEALTH_VPN_DURATION_INITIAL":  "HEALTH_VPN_DURATION_INITIAL is obsolete",
+		"HEALTH_VPN_DURATION_ADDITION": "HEALTH_VPN_DURATION_ADDITION is obsolete",
 	}
 	sortedKeys := maps.Keys(keyToMessage)
 	slices.Sort(sortedKeys)

internal/configuration/settings/health.go

@@ -2,6 +2,7 @@ package settings
 
 import (
 	"fmt"
+	"net/netip"
 	"os"
 	"time"
 
@@ -24,16 +25,13 @@ type Health struct {
 	// HTTP server. It defaults to 500 milliseconds.
 	ReadTimeout time.Duration
 	// TargetAddress is the address (host or host:port)
-	// to TCP dial to periodically for the health check.
+	// to TCP TLS dial to periodically for the health check.
 	// It cannot be the empty string in the internal state.
 	TargetAddress string
-	// SuccessWait is the duration to wait to re-run the
-	// healthcheck after a successful healthcheck.
-	// It defaults to 5 seconds and cannot be zero in
-	// the internal state.
-	SuccessWait time.Duration
-	// VPN has health settings specific to the VPN loop.
-	VPN HealthyWait
+	// ICMPTargetIP is the IP address to use for ICMP echo requests
+	// in the health checker. It can be set to an unspecified address
+	// such that the VPN server IP is used, which is also the default behavior.
+	ICMPTargetIP netip.Addr
 }
 
 func (h Health) Validate() (err error) {
@@ -42,11 +40,6 @@ func (h Health) Validate() (err error) {
 		return fmt.Errorf("server listening address is not valid: %w", err)
 	}
 
-	err = h.VPN.validate()
-	if err != nil {
-		return fmt.Errorf("health VPN settings: %w", err)
-	}
-
 	return nil
 }
 
@@ -56,8 +49,7 @@ func (h *Health) copy() (copied Health) {
 		ReadHeaderTimeout: h.ReadHeaderTimeout,
 		ReadTimeout:       h.ReadTimeout,
 		TargetAddress:     h.TargetAddress,
-		SuccessWait:       h.SuccessWait,
-		VPN:               h.VPN.copy(),
+		ICMPTargetIP:      h.ICMPTargetIP,
 	}
 }
 
@@ -69,8 +61,7 @@ func (h *Health) OverrideWith(other Health) {
 	h.ReadHeaderTimeout = gosettings.OverrideWithComparable(h.ReadHeaderTimeout, other.ReadHeaderTimeout)
 	h.ReadTimeout = gosettings.OverrideWithComparable(h.ReadTimeout, other.ReadTimeout)
 	h.TargetAddress = gosettings.OverrideWithComparable(h.TargetAddress, other.TargetAddress)
-	h.SuccessWait = gosettings.OverrideWithComparable(h.SuccessWait, other.SuccessWait)
-	h.VPN.overrideWith(other.VPN)
+	h.ICMPTargetIP = gosettings.OverrideWithComparable(h.ICMPTargetIP, other.ICMPTargetIP)
 }
 
 func (h *Health) SetDefaults() {
@@ -80,9 +71,7 @@ func (h *Health) SetDefaults() {
 	const defaultReadTimeout = 500 * time.Millisecond
 	h.ReadTimeout = gosettings.DefaultComparable(h.ReadTimeout, defaultReadTimeout)
 	h.TargetAddress = gosettings.DefaultComparable(h.TargetAddress, "cloudflare.com:443")
-	const defaultSuccessWait = 5 * time.Second
-	h.SuccessWait = gosettings.DefaultComparable(h.SuccessWait, defaultSuccessWait)
-	h.VPN.setDefaults()
+	h.ICMPTargetIP = gosettings.DefaultComparable(h.ICMPTargetIP, netip.IPv4Unspecified()) // use the VPN server IP
 }
 
 func (h Health) String() string {
@@ -93,27 +82,21 @@ func (h Health) toLinesNode() (node *gotree.Node) {
 	node = gotree.New("Health settings:")
 	node.Appendf("Server listening address: %s", h.ServerAddress)
 	node.Appendf("Target address: %s", h.TargetAddress)
-	node.Appendf("Duration to wait after success: %s", h.SuccessWait)
-	node.Appendf("Read header timeout: %s", h.ReadHeaderTimeout)
-	node.Appendf("Read timeout: %s", h.ReadTimeout)
-	node.AppendNode(h.VPN.toLinesNode("VPN"))
+	icmpTarget := "VPN server IP"
+	if !h.ICMPTargetIP.IsUnspecified() {
+		icmpTarget = h.ICMPTargetIP.String()
+	}
+	node.Appendf("ICMP target IP: %s", icmpTarget)
 	return node
 }
 
 func (h *Health) Read(r *reader.Reader) (err error) {
 	h.ServerAddress = r.String("HEALTH_SERVER_ADDRESS")
 	h.TargetAddress = r.String("HEALTH_TARGET_ADDRESS",
 		reader.RetroKeys("HEALTH_ADDRESS_TO_PING"))
-
-	h.SuccessWait, err = r.Duration("HEALTH_SUCCESS_WAIT_DURATION")
+	h.ICMPTargetIP, err = r.NetipAddr("HEALTH_ICMP_TARGET_IP")
 	if err != nil {
 		return err
 	}
-
-	err = h.VPN.read(r)
-	if err != nil {
-		return fmt.Errorf("VPN health settings: %w", err)
-	}
-
 	return nil
 }

internal/configuration/settings/healthywait.go

@@ -58,12 +58,7 @@ func Test_Settings_String(t *testing.T) {
 ├── Health settings:
 |   ├── Server listening address: 127.0.0.1:9999
 |   ├── Target address: cloudflare.com:443
-|   ├── Duration to wait after success: 5s
-|   ├── Read header timeout: 100ms
-|   ├── Read timeout: 500ms
-|   └── VPN wait durations:
-|       ├── Initial duration: 6s
-|       └── Additional duration: 5s
+|   └── ICMP target IP: VPN server IP
 ├── Shadowsocks server settings:
 |   └── Enabled: no
 ├── HTTP proxy settings: