improve proxy healthz checking

Author: stlaz

cmd/kube-rbac-proxy/app/kube-rbac-proxy.go

@@ -249,6 +249,7 @@ func Run(cfg *server.KubeRBACProxyConfig) error {
 		return err
 	}
 
+	var stoppedCh, listenerStoppedCh <-chan struct{}
 	go func() {
 		defer wg.Done()
 		defer cancel()
@@ -263,11 +264,15 @@ func Run(cfg *server.KubeRBACProxyConfig) error {
 		<-stoppedCh
 	}()
 
+	cfg.SecureServing.Cert.CurrentCertKeyContent()
+
 	if cfg.KubeRBACProxyInfo.ProxyEndpointsSecureServing != nil {
 		// we need a second listener in order to serve proxy-specific endpoints
 		// on a different port (--proxy-endpoints-port)
 		proxyEndpointsMux := http.NewServeMux()
-		proxyEndpointsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) })
+		proxyEndpointsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			proxyHealtzCheck(w, cfg.SecureServing.Listener.Addr().String(), listenerStoppedCh, stoppedCh)
+		})
 
 		if err := wg.Add(1); err != nil {
 			return err
@@ -354,3 +359,31 @@ func setupAuthorizer(krbInfo *server.KubeRBACProxyInfo, delegatedAuthz *serverco
 
 	return rewritingAuthorizer, nil
 }
+
+func proxyHealtzCheck(w http.ResponseWriter, localProxyAddr string, listenerStoppedChan, stoppedChan <-chan struct{}) {
+	select {
+	case <-stoppedChan:
+		http.Error(w, "the proxying port serving logic has stopped", http.StatusServiceUnavailable)
+		return
+	case <-listenerStoppedChan:
+		http.Error(w, "listener stopped", http.StatusServiceUnavailable)
+		return
+	default:
+	}
+
+	// we need the tls.Dialer otherwise the server would log EOF for TLS handshakes
+	// since the connection would be cut before that was ever attempted
+	dialer := tls.Dialer{NetDialer: &net.Dialer{}, Config: &tls.Config{InsecureSkipVerify: true}}
+	dialCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// we just knock on the other listener as we don't want to trigger proxying to upstream
+	conn, err := dialer.DialContext(dialCtx, "tcp", localProxyAddr)
+	if err != nil {
+		http.Error(w, "failed to connect to the proxying listener", http.StatusInternalServerError)
+		klog.Errorf("failed to connect to the proxying listener: %v", err)
+		return
+	}
+	_ = conn.Close()
+	_, _ = w.Write([]byte("ok"))
+}

test/e2e/basics.go

@@ -519,3 +519,47 @@ func testIgnorePaths(client kubernetes.Interface) kubetest.TestSuite {
 		}.Run(t)
 	}
 }
+
+func testHealthz(client kubernetes.Interface) kubetest.TestSuite {
+	return func(t *testing.T) {
+		command := `curl --connect-timeout 5 -v -s -k --fail https://kube-rbac-proxy.default.svc.cluster.local:8643/healthz`
+
+		kubetest.Scenario{
+			Name:        "healtz check",
+			Description: "check that proxy /healthz works as expected",
+
+			Given: kubetest.Actions(
+				kubetest.CreatedManifests(
+					client,
+					"basics/clusterRole.yaml",
+					"basics/clusterRoleBinding.yaml",
+					"basics/deployment.yaml",
+					"basics/service.yaml",
+					"basics/serviceAccount.yaml",
+				),
+			),
+			When: kubetest.Actions(
+				kubetest.PodsAreReady(
+					client,
+					1,
+					"app=kube-rbac-proxy",
+				),
+				kubetest.ServiceIsReady(
+					client,
+					"kube-rbac-proxy",
+				),
+			),
+			Then: kubetest.Actions(
+				kubetest.ClientLogsContain(
+					client,
+					command,
+					[]string{
+						"{ [2 bytes data]",
+						"ok",
+					},
+					nil,
+				),
+			),
+		}.Run(t)
+	}
+}

test/e2e/basics/deployment.yaml

@@ -19,12 +19,20 @@ spec:
           image: quay.io/brancz/kube-rbac-proxy:local
           args:
             - "--secure-port=8443"
+            - "--proxy-endpoints-port=8643"
             - "--upstream=http://127.0.0.1:8081/"
             - "--authentication-skip-lookup"
             - "--v=10"
           ports:
             - containerPort: 8443
               name: https
+            - containerPort: 8643
+              name: proxy
+          readinessProbe:
+            httpGet:
+              scheme: HTTPS
+              port: 8643
+              path: healthz
         - name: prometheus-example-app
           image: quay.io/brancz/prometheus-example-app:v0.1.0
           args:

test/e2e/basics/service.yaml

@@ -10,5 +10,8 @@ spec:
     - name: https
       port: 8443
       targetPort: https
+    - name: proxy
+      port: 8643
+      targetPort: proxy
   selector:
     app: kube-rbac-proxy

test/e2e/main_test.go

@@ -52,6 +52,7 @@ func TestMain(m *testing.M) {
 func Test(t *testing.T) {
 	tests := map[string]kubetest.TestSuite{
 		"Basics":             testBasics(client),
+		"Healthz":            testHealthz(client),
 		"H2CUpstream":        testH2CUpstream(client),
 		"IdentityHeaders":    testIdentityHeaders(client),
 		"ClientCertificates": testClientCertificates(client),