From a67157906d4d3a62908ebc8a93f0bc9b1eef693b Mon Sep 17 00:00:00 2001 From: Sandhya Dasu Date: Thu, 30 Nov 2023 10:24:46 -0500 Subject: [PATCH] Add the ability to render and monitor Cloud LB IPs On some cloud platforms, the user cannot use the cloud's default DNS solution. They are expected to use their own custom DNS solution that is external to the cluster. OpenShift is not allowed to configure this DNS solution. In this scenario, these same customers want to continue using the cloud provided Load Balancers (LBs). OpenShift is expected to continue configuing the cloud LBs for API, API-Int and Ingress access. Since the LB information is not available, before cluster installation, the user cannot configure their custom DNS solution before cluster installation. So, to support this mode, OpenShift needs to start its in-cluster CoreDNS based DNS solution for API, API-Int and Ingress resolution so that cluster installation is successful. The customer is expected to configure their DNS solution post-install. At this the time, the cloud's API and API-Int LBs are configured by the Installer and its value is not expected to change during the life of the cluster. The Ingress operator continues to handle Ingress LB configuration. Based on enhancement: openshift/enhancements#1468 --- cmd/corednsmonitor/corednsmonitor.go | 14 ++- cmd/runtimecfg/display.go | 20 +++- cmd/runtimecfg/render.go | 19 +++- pkg/config/cloud_lb_config_test.go | 45 +++++++++ pkg/config/node.go | 144 ++++++++++++++++++++++----- pkg/monitor/corednsmonitor.go | 9 +- pkg/monitor/dnsmasqmonitor.go | 2 +- pkg/monitor/dynkeepalived.go | 4 +- 8 files changed, 225 insertions(+), 32 deletions(-) create mode 100644 pkg/config/cloud_lb_config_test.go diff --git a/cmd/corednsmonitor/corednsmonitor.go b/cmd/corednsmonitor/corednsmonitor.go index e5f8680a..66a2fd6d 100644 --- a/cmd/corednsmonitor/corednsmonitor.go +++ b/cmd/corednsmonitor/corednsmonitor.go @@ -55,8 +55,20 @@ func main() { if err != nil { return err } + cloudExtLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + cloudExtLBIPs = []net.IP{} + } + cloudIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + cloudIntLBIPs = []net.IP{} + } + cloudIngressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips") + if err != nil { + cloudIngressLBIPs = []net.IP{} + } - return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval) + return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval, cloudExtLBIPs, cloudIntLBIPs, cloudIngressLBIPs) }, } rootCmd.PersistentFlags().StringP("cluster-config", "c", "", "Path to cluster-config ConfigMap to retrieve ControlPlane info") diff --git a/cmd/runtimecfg/display.go b/cmd/runtimecfg/display.go index 97134a63..a03ef2a0 100644 --- a/cmd/runtimecfg/display.go +++ b/cmd/runtimecfg/display.go @@ -28,6 +28,9 @@ func init() { displayCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at") displayCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at") displayCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers") + displayCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API") + displayCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift API") + displayCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers") rootCmd.AddCommand(displayCmd) } @@ -84,7 +87,22 @@ func runDisplay(cmd *cobra.Command, args []string) error { if err != nil { return err } - config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort) + + apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + apiLBIPs = []net.IP{} + } + apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + apiIntLBIPs = []net.IP{} + } + ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips") + if err != nil { + ingressLBIPs = []net.IP{} + } + clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs} + + config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig) if err != nil { return err } diff --git a/cmd/runtimecfg/render.go b/cmd/runtimecfg/render.go index 528e831f..e443a63f 100644 --- a/cmd/runtimecfg/render.go +++ b/cmd/runtimecfg/render.go @@ -33,6 +33,9 @@ func init() { renderCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at") renderCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at") renderCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers") + renderCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API") + renderCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift Internal API") + renderCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers") rootCmd.AddCommand(renderCmd) } @@ -88,7 +91,21 @@ func runRender(cmd *cobra.Command, args []string) error { if err != nil { return err } - config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort) + + apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + apiLBIPs = []net.IP{} + } + apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + apiIntLBIPs = []net.IP{} + } + ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips") + if err != nil { + ingressLBIPs = []net.IP{} + } + clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs} + config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig) if err != nil { return err } diff --git a/pkg/config/cloud_lb_config_test.go b/pkg/config/cloud_lb_config_test.go new file mode 100644 index 00000000..c380ea5e --- /dev/null +++ b/pkg/config/cloud_lb_config_test.go @@ -0,0 +1,45 @@ +package config + +import ( + "net" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + testKubeconfigPath = "/test/path/kubeconfig" + testClusterConfigPath = "/test/path/clusterConfig" + testResolvConfPath = "/test/path/resolvConf" + testApiLBIPv4 = net.ParseIP("192.168.0.111") + testApiIntLBIPv4 = net.ParseIP("10.10.10.20") + testIngressOneIPv4 = net.ParseIP("192.168.20.140") + testIngressTwoIPv4 = net.ParseIP("10.10.10.40") + testClusterLBConfig = ClusterLBConfig{ + ApiLBIPs: []net.IP{testApiLBIPv4}, + ApiIntLBIPs: []net.IP{testApiIntLBIPv4}, + IngressLBIPs: []net.IP{testIngressOneIPv4, testIngressTwoIPv4}} + expectedApiLBIPv4 = "192.168.0.111" + expectedIngressTwoIPv4 = "10.10.10.40" + testNode = Node{} +) + +var _ = Describe("PopulateCloudLBIPAddresses", func() { + Context("for IPV4 Cloud LB IPs", func() { + Context("with multiple Ingress LB IPs", func() { + It("matches multiple IPs in 1 node", func() { + testNode, err := PopulateCloudLBIPAddresses(testClusterLBConfig, testNode) + Expect(testNode.Cluster.APILBIPs[0]).To(Equal(expectedApiLBIPv4)) + Expect(testNode.Cluster.IngressLBIPs[1]).To(Equal(expectedIngressTwoIPv4)) + Expect(testNode.Cluster.CloudLBRecordType).To(Equal("A")) + Expect(err).To(BeNil()) + }) + }) + }) +}) + +func TestCloudLBConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cloud LB Config Tests") +} diff --git a/pkg/config/node.go b/pkg/config/node.go index 84ebb396..1404041f 100644 --- a/pkg/config/node.go +++ b/pkg/config/node.go @@ -52,6 +52,11 @@ type Cluster struct { VIPNetmask int MasterAmount int64 NodeAddresses []NodeAddress + APILBIPs []string + APIIntLBIPs []string + IngressLBIPs []string + CloudLBRecordType string + CloudLBEmptyType string } type Backend struct { @@ -84,6 +89,12 @@ type Node struct { Configs *[]Node } +type ClusterLBConfig struct { + ApiLBIPs []net.IP + ApiIntLBIPs []net.IP + IngressLBIPs []net.IP +} + func getDNSUpstreams(resolvConfPath string) (upstreams []string, err error) { dnsFile, err := os.Open(resolvConfPath) if err != nil { @@ -417,34 +428,42 @@ func getNodeIpForRequestedIpStack(node v1.Node, filterIps []string, machineNetwo // apiPort: The port on which the k8s api listens. Should be 6443. // lbPort: The port on which haproxy listens. // statPort: The port on which the haproxy stats endpoint listens. -func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) { - vipCount := 0 - if len(apiVips) > len(ingressVips) { - vipCount = len(apiVips) - } else { - vipCount = len(ingressVips) - } - nodes := []Node{} - var apiVip, ingressVip net.IP - for i := 0; i < vipCount; i++ { - if i < len(apiVips) { - apiVip = apiVips[i] - } else { - apiVip = nil - } - if i < len(ingressVips) { - ingressVip = ingressVips[i] +// clusterLBConfig: A struct containing IPs for API, API-Int and Ingress LBs +func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16, clusterLBConfig ClusterLBConfig) (node Node, err error) { + switch { + case len(clusterLBConfig.ApiIntLBIPs) > 0: + // Cloud Platforms with cloud LBs but no Cloud DNS + return getConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath, clusterLBConfig) + default: + // On-prem platforms + vipCount := 0 + if len(apiVips) > len(ingressVips) { + vipCount = len(apiVips) } else { - ingressVip = nil + vipCount = len(ingressVips) } - newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort) - if err != nil { - return Node{}, err + nodes := []Node{} + var apiVip, ingressVip net.IP + for i := 0; i < vipCount; i++ { + if i < len(apiVips) { + apiVip = apiVips[i] + } else { + apiVip = nil + } + if i < len(ingressVips) { + ingressVip = ingressVips[i] + } else { + ingressVip = nil + } + newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort) + if err != nil { + return Node{}, err + } + nodes = append(nodes, newNode) } - nodes = append(nodes, newNode) + nodes[0].Configs = &nodes + return nodes[0], nil } - nodes[0].Configs = &nodes - return nodes[0], nil } func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) { @@ -694,3 +713,80 @@ func PopulateNodeAddresses(kubeconfigPath string, node *Node) { } } } + +func getConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath string, clusterLBConfig ClusterLBConfig) (node Node, err error) { + var apiLBIP, apiIntLBIP, ingressIP net.IP + nodes := []Node{} + ipCount := 0 + if len(clusterLBConfig.ApiIntLBIPs) > len(clusterLBConfig.IngressLBIPs) { + ipCount = len(clusterLBConfig.ApiIntLBIPs) + } else { + ipCount = len(clusterLBConfig.IngressLBIPs) + } + for i := 0; i < ipCount; i++ { + if i < len(clusterLBConfig.ApiIntLBIPs) { + apiIntLBIP = clusterLBConfig.ApiIntLBIPs[i] + } else { + apiIntLBIP = nil + } + + // For public clusters. Private clusters will not have External + // LBs so apiLBIPs could be empty. + if len(clusterLBConfig.ApiLBIPs) != 0 && i < len(clusterLBConfig.ApiLBIPs) { + apiLBIP = clusterLBConfig.ApiLBIPs[i] + } else { + apiLBIP = nil + } + if len(clusterLBConfig.IngressLBIPs) != 0 && i < len(clusterLBConfig.IngressLBIPs) { + ingressIP = clusterLBConfig.IngressLBIPs[i] + } else { + ingressIP = nil + } + newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, nil, nil, 0, 0, 0) + if err != nil { + return Node{}, err + } + newNode = updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP, newNode) + nodes = append(nodes, newNode) + } + nodes[0].Configs = &nodes + return nodes[0], nil +} + +func updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP net.IP, node Node) Node { + if apiIntLBIP != nil { + node.Cluster.CloudLBRecordType = "A" + node.Cluster.CloudLBEmptyType = "AAAA" + if apiIntLBIP.To4() == nil { + node.Cluster.CloudLBRecordType = "AAAA" + node.Cluster.CloudLBEmptyType = "A" + } + node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIP.String()) + } + if apiLBIP != nil { + node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIP.String()) + } + if ingressIP != nil { + node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ingressIP.String()) + } + return node +} + +func PopulateCloudLBIPAddresses(clusterLBConfig ClusterLBConfig, node Node) (updatedNode Node, err error) { + for _, ip := range clusterLBConfig.ApiIntLBIPs { + node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, ip.String()) + } + for _, ip := range clusterLBConfig.ApiLBIPs { + node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, ip.String()) + } + for _, ip := range clusterLBConfig.IngressLBIPs { + node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ip.String()) + } + node.Cluster.CloudLBRecordType = "A" + node.Cluster.CloudLBEmptyType = "AAAA" + if len(clusterLBConfig.ApiIntLBIPs) > 0 && clusterLBConfig.ApiIntLBIPs[0].To4() == nil { + node.Cluster.CloudLBRecordType = "AAAA" + node.Cluster.CloudLBEmptyType = "A" + } + return node, nil +} diff --git a/pkg/monitor/corednsmonitor.go b/pkg/monitor/corednsmonitor.go index fcb80b9a..7da36847 100644 --- a/pkg/monitor/corednsmonitor.go +++ b/pkg/monitor/corednsmonitor.go @@ -16,7 +16,7 @@ import ( const resolvConfFilepath string = "/var/run/NetworkManager/resolv.conf" -func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration) error { +func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration, apiLBIPs, apiIntLBIPs, ingressLBIPs []net.IP) error { signals := make(chan os.Signal, 1) done := make(chan bool, 1) @@ -42,10 +42,15 @@ func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath strin if err != nil { return err } - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0) + clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs} + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0, clusterLBConfig) if err != nil { return err } + // Populate cloud LB IP addresses for platforms where the cloud LBs + // have already been configured + newConfig, _ = config.PopulateCloudLBIPAddresses(clusterLBConfig, newConfig) + config.PopulateNodeAddresses(kubeconfigPath, &newConfig) // There should never be 0 nodes in a functioning cluster. This means // we failed to populate the list, so we don't want to render. diff --git a/pkg/monitor/dnsmasqmonitor.go b/pkg/monitor/dnsmasqmonitor.go index 5371049a..4133a197 100644 --- a/pkg/monitor/dnsmasqmonitor.go +++ b/pkg/monitor/dnsmasqmonitor.go @@ -33,7 +33,7 @@ func DnsmasqWatch(kubeconfigPath, templatePath, cfgPath string, apiVips []net.IP return nil default: // We only care about the api vip and cluster domain here - config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0) + config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0, config.ClusterLBConfig{}) if err != nil { return err } diff --git a/pkg/monitor/dynkeepalived.go b/pkg/monitor/dynkeepalived.go index c143d870..f6865d94 100644 --- a/pkg/monitor/dynkeepalived.go +++ b/pkg/monitor/dynkeepalived.go @@ -355,7 +355,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st case desiredModeInfo := <-updateModeCh: - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0) + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, config.ClusterLBConfig{}) if err != nil { return err } @@ -437,7 +437,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st } } } - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0) + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, config.ClusterLBConfig{}) if err != nil { return err }