discovery: cap health check reconnect backoff at 10s (#19967)

Signed-off-by: Gwanho Kim <khkim6040@gmail.com>

Author: khkim6040

go/vt/discovery/tablet_health_check.go

@@ -323,15 +323,25 @@ func (thc *tabletHealthCheck) checkConn(hc *HealthCheckImpl) {
 			return
 		case <-time.After(retryDelay):
 			// Exponentially back-off to prevent tight-loop.
-			retryDelay *= 2
-			// Limit the retry delay backoff to the health check timeout
-			if retryDelay > hc.healthCheckTimeout {
-				retryDelay = hc.healthCheckTimeout
-			}
+			retryDelay = nextHealthCheckRetryDelay(retryDelay)
 		}
 	}
 }
 
+// maxHealthCheckRetryDelay caps the exponential back-off between healthcheck
+// reconnection attempts. Capping well below healthCheckTimeout (default 1m)
+// ensures vtgate rediscovers a recovered tablet promptly instead of sleeping
+// out a back-off that had grown to the silence timeout. See #19894.
+const maxHealthCheckRetryDelay = 10 * time.Second
+
+// nextHealthCheckRetryDelay doubles the current back-off delay, capping it at
+// maxHealthCheckRetryDelay. A delay already above the cap (operator-configured)
+// is left unchanged, never reduced. See #19894.
+func nextHealthCheckRetryDelay(current time.Duration) time.Duration {
+	next := current * 2
+	return min(next, max(current, maxHealthCheckRetryDelay))
+}
+
 func (thc *tabletHealthCheck) closeConnection(ctx context.Context, err error) {
 	thc.logger.Warningf("tablet %v healthcheck stream error: %v", thc.Tablet, err)
 	thc.setServingState(false, err.Error())

go/vt/discovery/tablet_health_check_test.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2026 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestNextHealthCheckRetryDelay verifies the back-off doubles below the cap,
+// caps at maxHealthCheckRetryDelay, and leaves an already-above-cap delay
+// unchanged rather than reducing it. See #19894.
+func TestNextHealthCheckRetryDelay(t *testing.T) {
+	// Doubles while comfortably below the cap.
+	assert.Equal(t, 20*time.Millisecond, nextHealthCheckRetryDelay(10*time.Millisecond))
+	assert.Equal(t, 2*time.Second, nextHealthCheckRetryDelay(time.Second))
+
+	// Caps at maxHealthCheckRetryDelay once doubling would exceed it.
+	assert.Equal(t, maxHealthCheckRetryDelay, nextHealthCheckRetryDelay(6*time.Second))
+	assert.Equal(t, maxHealthCheckRetryDelay, nextHealthCheckRetryDelay(maxHealthCheckRetryDelay))
+
+	// A delay already above the cap is left unchanged, not reduced.
+	assert.Equal(t, time.Hour, nextHealthCheckRetryDelay(time.Hour))
+
+	// The cap stays well under the default health check timeout, so a recovered
+	// tablet is not stranded behind a back-off grown to the silence timeout.
+	assert.Less(t, maxHealthCheckRetryDelay, DefaultHealthCheckTimeout)
+}