From d9d3b889d9d4511de14845e716e9ff3afe993377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanislav=20L=C3=A1zni=C4=8Dka?= Date: Wed, 13 Nov 2024 12:58:51 +0100 Subject: [PATCH] improve proxy healthz checking --- cmd/kube-rbac-proxy/app/kube-rbac-proxy.go | 35 ++++++++++++++++- test/e2e/basics.go | 44 ++++++++++++++++++++++ test/e2e/basics/deployment.yaml | 8 ++++ test/e2e/basics/service.yaml | 3 ++ test/e2e/main_test.go | 1 + 5 files changed, 89 insertions(+), 2 deletions(-) diff --git a/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go b/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go index 4a58d781f..16638d61a 100644 --- a/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go +++ b/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go @@ -226,11 +226,12 @@ func Run(cfg *server.KubeRBACProxyConfig) error { return err } + var stoppedCh, listenerStoppedCh <-chan struct{} go func() { defer wg.Done() defer cancel() - stoppedCh, listenerStoppedCh, err := cfg.SecureServing.Serve(mux, 10*time.Second, serverCtx.Done()) + stoppedCh, listenerStoppedCh, err = cfg.SecureServing.Serve(mux, 10*time.Second, serverCtx.Done()) if err != nil { klog.Errorf("%v", err) return @@ -244,7 +245,9 @@ func Run(cfg *server.KubeRBACProxyConfig) error { // we need a second listener in order to serve proxy-specific endpoints // on a different port (--proxy-endpoints-port) proxyEndpointsMux := http.NewServeMux() - proxyEndpointsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) }) + proxyEndpointsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + proxyHealtzCheck(w, cfg.SecureServing.Listener.Addr().String(), listenerStoppedCh, stoppedCh) + }) if err := wg.Add(1); err != nil { return err @@ -376,3 +379,31 @@ func copyHeaderIfSet(inReq *http.Request, outReq *http.Request, headerKey string outReq.Header.Add(headerKey, v) } } + +func proxyHealtzCheck(w http.ResponseWriter, localProxyAddr string, listenerStoppedChan, stoppedChan <-chan struct{}) { + select { + case <-stoppedChan: + http.Error(w, "the proxying port serving logic has stopped", http.StatusServiceUnavailable) + return + case <-listenerStoppedChan: + http.Error(w, "listener stopped", http.StatusServiceUnavailable) + return + default: + } + + // we need the tls.Dialer otherwise the server would log EOF for TLS handshakes + // since the connection would be cut before that was ever attempted + dialer := tls.Dialer{NetDialer: &net.Dialer{}, Config: &tls.Config{InsecureSkipVerify: true}} // the cert's not likely to have loopback IP SAN + dialCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // we just knock on the other listener as we don't want to trigger proxying to upstream + conn, err := dialer.DialContext(dialCtx, "tcp", localProxyAddr) + if err != nil { + http.Error(w, "failed to connect to the proxying listener", http.StatusInternalServerError) + klog.Errorf("failed to connect to the proxying listener: %v", err) + return + } + _ = conn.Close() + _, _ = w.Write([]byte("ok")) +} diff --git a/test/e2e/basics.go b/test/e2e/basics.go index 0e1da923c..a0a5271f7 100644 --- a/test/e2e/basics.go +++ b/test/e2e/basics.go @@ -519,3 +519,47 @@ func testIgnorePaths(client kubernetes.Interface) kubetest.TestSuite { }.Run(t) } } + +func testHealthz(client kubernetes.Interface) kubetest.TestSuite { + return func(t *testing.T) { + command := `curl --connect-timeout 5 -v -s -k --fail https://kube-rbac-proxy.default.svc.cluster.local:8643/healthz` + + kubetest.Scenario{ + Name: "healtz check", + Description: "check that proxy /healthz works as expected", + + Given: kubetest.Actions( + kubetest.CreatedManifests( + client, + "basics/clusterRole.yaml", + "basics/clusterRoleBinding.yaml", + "basics/deployment.yaml", + "basics/service.yaml", + "basics/serviceAccount.yaml", + ), + ), + When: kubetest.Actions( + kubetest.PodsAreReady( + client, + 1, + "app=kube-rbac-proxy", + ), + kubetest.ServiceIsReady( + client, + "kube-rbac-proxy", + ), + ), + Then: kubetest.Actions( + kubetest.ClientLogsContain( + client, + command, + []string{ + "{ [2 bytes data]", + "ok", + }, + nil, + ), + ), + }.Run(t) + } +} diff --git a/test/e2e/basics/deployment.yaml b/test/e2e/basics/deployment.yaml index 20ea3fe46..7bc11dd12 100644 --- a/test/e2e/basics/deployment.yaml +++ b/test/e2e/basics/deployment.yaml @@ -19,12 +19,20 @@ spec: image: quay.io/brancz/kube-rbac-proxy:local args: - "--secure-port=8443" + - "--proxy-endpoints-port=8643" - "--upstream=http://127.0.0.1:8081/" - "--authentication-skip-lookup" - "--v=10" ports: - containerPort: 8443 name: https + - containerPort: 8643 + name: proxy + readinessProbe: + httpGet: + scheme: HTTPS + port: 8643 + path: healthz - name: prometheus-example-app image: quay.io/brancz/prometheus-example-app:v0.1.0 args: diff --git a/test/e2e/basics/service.yaml b/test/e2e/basics/service.yaml index b1ae11686..5eb463563 100644 --- a/test/e2e/basics/service.yaml +++ b/test/e2e/basics/service.yaml @@ -10,5 +10,8 @@ spec: - name: https port: 8443 targetPort: https + - name: proxy + port: 8643 + targetPort: proxy selector: app: kube-rbac-proxy diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 1e6622680..586689684 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -52,6 +52,7 @@ func TestMain(m *testing.M) { func Test(t *testing.T) { tests := map[string]kubetest.TestSuite{ "Basics": testBasics(client), + "Healthz": testHealthz(client), "H2CUpstream": testH2CUpstream(client), "IdentityHeaders": testIdentityHeaders(client), "ClientCertificates": testClientCertificates(client),