diff --git a/cmd/corednsmonitor/corednsmonitor.go b/cmd/corednsmonitor/corednsmonitor.go index e5f8680a..66a2fd6d 100644 --- a/cmd/corednsmonitor/corednsmonitor.go +++ b/cmd/corednsmonitor/corednsmonitor.go @@ -55,8 +55,20 @@ func main() { if err != nil { return err } + cloudExtLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + cloudExtLBIPs = []net.IP{} + } + cloudIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + cloudIntLBIPs = []net.IP{} + } + cloudIngressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips") + if err != nil { + cloudIngressLBIPs = []net.IP{} + } - return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval) + return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval, cloudExtLBIPs, cloudIntLBIPs, cloudIngressLBIPs) }, } rootCmd.PersistentFlags().StringP("cluster-config", "c", "", "Path to cluster-config ConfigMap to retrieve ControlPlane info") diff --git a/cmd/runtimecfg/display.go b/cmd/runtimecfg/display.go index 97134a63..a03ef2a0 100644 --- a/cmd/runtimecfg/display.go +++ b/cmd/runtimecfg/display.go @@ -28,6 +28,9 @@ func init() { displayCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at") displayCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at") displayCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers") + displayCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API") + displayCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift API") + displayCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers") rootCmd.AddCommand(displayCmd) } @@ -84,7 +87,22 @@ func runDisplay(cmd *cobra.Command, args []string) error { if err != nil { return err } - config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort) + + apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + apiLBIPs = []net.IP{} + } + apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + apiIntLBIPs = []net.IP{} + } + ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips") + if err != nil { + ingressLBIPs = []net.IP{} + } + clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs} + + config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig) if err != nil { return err } diff --git a/cmd/runtimecfg/render.go b/cmd/runtimecfg/render.go index 528e831f..e443a63f 100644 --- a/cmd/runtimecfg/render.go +++ b/cmd/runtimecfg/render.go @@ -33,6 +33,9 @@ func init() { renderCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at") renderCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at") renderCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers") + renderCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API") + renderCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift Internal API") + renderCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers") rootCmd.AddCommand(renderCmd) } @@ -88,7 +91,21 @@ func runRender(cmd *cobra.Command, args []string) error { if err != nil { return err } - config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort) + + apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + apiLBIPs = []net.IP{} + } + apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + apiIntLBIPs = []net.IP{} + } + ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips") + if err != nil { + ingressLBIPs = []net.IP{} + } + clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs} + config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig) if err != nil { return err } diff --git a/pkg/config/cloud_lb_config_test.go b/pkg/config/cloud_lb_config_test.go new file mode 100644 index 00000000..c380ea5e --- /dev/null +++ b/pkg/config/cloud_lb_config_test.go @@ -0,0 +1,45 @@ +package config + +import ( + "net" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + testKubeconfigPath = "/test/path/kubeconfig" + testClusterConfigPath = "/test/path/clusterConfig" + testResolvConfPath = "/test/path/resolvConf" + testApiLBIPv4 = net.ParseIP("192.168.0.111") + testApiIntLBIPv4 = net.ParseIP("10.10.10.20") + testIngressOneIPv4 = net.ParseIP("192.168.20.140") + testIngressTwoIPv4 = net.ParseIP("10.10.10.40") + testClusterLBConfig = ClusterLBConfig{ + ApiLBIPs: []net.IP{testApiLBIPv4}, + ApiIntLBIPs: []net.IP{testApiIntLBIPv4}, + IngressLBIPs: []net.IP{testIngressOneIPv4, testIngressTwoIPv4}} + expectedApiLBIPv4 = "192.168.0.111" + expectedIngressTwoIPv4 = "10.10.10.40" + testNode = Node{} +) + +var _ = Describe("PopulateCloudLBIPAddresses", func() { + Context("for IPV4 Cloud LB IPs", func() { + Context("with multiple Ingress LB IPs", func() { + It("matches multiple IPs in 1 node", func() { + testNode, err := PopulateCloudLBIPAddresses(testClusterLBConfig, testNode) + Expect(testNode.Cluster.APILBIPs[0]).To(Equal(expectedApiLBIPv4)) + Expect(testNode.Cluster.IngressLBIPs[1]).To(Equal(expectedIngressTwoIPv4)) + Expect(testNode.Cluster.CloudLBRecordType).To(Equal("A")) + Expect(err).To(BeNil()) + }) + }) + }) +}) + +func TestCloudLBConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cloud LB Config Tests") +} diff --git a/pkg/config/node.go b/pkg/config/node.go index 84ebb396..1404041f 100644 --- a/pkg/config/node.go +++ b/pkg/config/node.go @@ -52,6 +52,11 @@ type Cluster struct { VIPNetmask int MasterAmount int64 NodeAddresses []NodeAddress + APILBIPs []string + APIIntLBIPs []string + IngressLBIPs []string + CloudLBRecordType string + CloudLBEmptyType string } type Backend struct { @@ -84,6 +89,12 @@ type Node struct { Configs *[]Node } +type ClusterLBConfig struct { + ApiLBIPs []net.IP + ApiIntLBIPs []net.IP + IngressLBIPs []net.IP +} + func getDNSUpstreams(resolvConfPath string) (upstreams []string, err error) { dnsFile, err := os.Open(resolvConfPath) if err != nil { @@ -417,34 +428,42 @@ func getNodeIpForRequestedIpStack(node v1.Node, filterIps []string, machineNetwo // apiPort: The port on which the k8s api listens. Should be 6443. // lbPort: The port on which haproxy listens. // statPort: The port on which the haproxy stats endpoint listens. -func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) { - vipCount := 0 - if len(apiVips) > len(ingressVips) { - vipCount = len(apiVips) - } else { - vipCount = len(ingressVips) - } - nodes := []Node{} - var apiVip, ingressVip net.IP - for i := 0; i < vipCount; i++ { - if i < len(apiVips) { - apiVip = apiVips[i] - } else { - apiVip = nil - } - if i < len(ingressVips) { - ingressVip = ingressVips[i] +// clusterLBConfig: A struct containing IPs for API, API-Int and Ingress LBs +func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16, clusterLBConfig ClusterLBConfig) (node Node, err error) { + switch { + case len(clusterLBConfig.ApiIntLBIPs) > 0: + // Cloud Platforms with cloud LBs but no Cloud DNS + return getConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath, clusterLBConfig) + default: + // On-prem platforms + vipCount := 0 + if len(apiVips) > len(ingressVips) { + vipCount = len(apiVips) } else { - ingressVip = nil + vipCount = len(ingressVips) } - newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort) - if err != nil { - return Node{}, err + nodes := []Node{} + var apiVip, ingressVip net.IP + for i := 0; i < vipCount; i++ { + if i < len(apiVips) { + apiVip = apiVips[i] + } else { + apiVip = nil + } + if i < len(ingressVips) { + ingressVip = ingressVips[i] + } else { + ingressVip = nil + } + newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort) + if err != nil { + return Node{}, err + } + nodes = append(nodes, newNode) } - nodes = append(nodes, newNode) + nodes[0].Configs = &nodes + return nodes[0], nil } - nodes[0].Configs = &nodes - return nodes[0], nil } func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) { @@ -694,3 +713,80 @@ func PopulateNodeAddresses(kubeconfigPath string, node *Node) { } } } + +func getConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath string, clusterLBConfig ClusterLBConfig) (node Node, err error) { + var apiLBIP, apiIntLBIP, ingressIP net.IP + nodes := []Node{} + ipCount := 0 + if len(clusterLBConfig.ApiIntLBIPs) > len(clusterLBConfig.IngressLBIPs) { + ipCount = len(clusterLBConfig.ApiIntLBIPs) + } else { + ipCount = len(clusterLBConfig.IngressLBIPs) + } + for i := 0; i < ipCount; i++ { + if i < len(clusterLBConfig.ApiIntLBIPs) { + apiIntLBIP = clusterLBConfig.ApiIntLBIPs[i] + } else { + apiIntLBIP = nil + } + + // For public clusters. Private clusters will not have External + // LBs so apiLBIPs could be empty. + if len(clusterLBConfig.ApiLBIPs) != 0 && i < len(clusterLBConfig.ApiLBIPs) { + apiLBIP = clusterLBConfig.ApiLBIPs[i] + } else { + apiLBIP = nil + } + if len(clusterLBConfig.IngressLBIPs) != 0 && i < len(clusterLBConfig.IngressLBIPs) { + ingressIP = clusterLBConfig.IngressLBIPs[i] + } else { + ingressIP = nil + } + newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, nil, nil, 0, 0, 0) + if err != nil { + return Node{}, err + } + newNode = updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP, newNode) + nodes = append(nodes, newNode) + } + nodes[0].Configs = &nodes + return nodes[0], nil +} + +func updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP net.IP, node Node) Node { + if apiIntLBIP != nil { + node.Cluster.CloudLBRecordType = "A" + node.Cluster.CloudLBEmptyType = "AAAA" + if apiIntLBIP.To4() == nil { + node.Cluster.CloudLBRecordType = "AAAA" + node.Cluster.CloudLBEmptyType = "A" + } + node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIP.String()) + } + if apiLBIP != nil { + node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIP.String()) + } + if ingressIP != nil { + node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ingressIP.String()) + } + return node +} + +func PopulateCloudLBIPAddresses(clusterLBConfig ClusterLBConfig, node Node) (updatedNode Node, err error) { + for _, ip := range clusterLBConfig.ApiIntLBIPs { + node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, ip.String()) + } + for _, ip := range clusterLBConfig.ApiLBIPs { + node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, ip.String()) + } + for _, ip := range clusterLBConfig.IngressLBIPs { + node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ip.String()) + } + node.Cluster.CloudLBRecordType = "A" + node.Cluster.CloudLBEmptyType = "AAAA" + if len(clusterLBConfig.ApiIntLBIPs) > 0 && clusterLBConfig.ApiIntLBIPs[0].To4() == nil { + node.Cluster.CloudLBRecordType = "AAAA" + node.Cluster.CloudLBEmptyType = "A" + } + return node, nil +} diff --git a/pkg/monitor/corednsmonitor.go b/pkg/monitor/corednsmonitor.go index fcb80b9a..7da36847 100644 --- a/pkg/monitor/corednsmonitor.go +++ b/pkg/monitor/corednsmonitor.go @@ -16,7 +16,7 @@ import ( const resolvConfFilepath string = "/var/run/NetworkManager/resolv.conf" -func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration) error { +func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration, apiLBIPs, apiIntLBIPs, ingressLBIPs []net.IP) error { signals := make(chan os.Signal, 1) done := make(chan bool, 1) @@ -42,10 +42,15 @@ func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath strin if err != nil { return err } - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0) + clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs} + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0, clusterLBConfig) if err != nil { return err } + // Populate cloud LB IP addresses for platforms where the cloud LBs + // have already been configured + newConfig, _ = config.PopulateCloudLBIPAddresses(clusterLBConfig, newConfig) + config.PopulateNodeAddresses(kubeconfigPath, &newConfig) // There should never be 0 nodes in a functioning cluster. This means // we failed to populate the list, so we don't want to render. diff --git a/pkg/monitor/dnsmasqmonitor.go b/pkg/monitor/dnsmasqmonitor.go index 5371049a..4133a197 100644 --- a/pkg/monitor/dnsmasqmonitor.go +++ b/pkg/monitor/dnsmasqmonitor.go @@ -33,7 +33,7 @@ func DnsmasqWatch(kubeconfigPath, templatePath, cfgPath string, apiVips []net.IP return nil default: // We only care about the api vip and cluster domain here - config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0) + config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0, config.ClusterLBConfig{}) if err != nil { return err } diff --git a/pkg/monitor/dynkeepalived.go b/pkg/monitor/dynkeepalived.go index c143d870..f6865d94 100644 --- a/pkg/monitor/dynkeepalived.go +++ b/pkg/monitor/dynkeepalived.go @@ -355,7 +355,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st case desiredModeInfo := <-updateModeCh: - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0) + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, config.ClusterLBConfig{}) if err != nil { return err } @@ -437,7 +437,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st } } } - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0) + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, config.ClusterLBConfig{}) if err != nil { return err }