Add the ability to render and monitor Cloud LB IPs

On some cloud platforms, the user cannot use the cloud's
default DNS solution. They are expected to use their own
custom DNS solution that is external to the cluster. OpenShift
is not allowed to configure this DNS solution. In this scenario,
these same customers want to continue using the cloud provided
Load Balancers (LBs).

OpenShift is expected to continue configuing the cloud LBs for
API, API-Int and Ingress access. Since the LB information is not
available, before cluster installation, the user cannot configure
their custom DNS solution before cluster installation.

So, to support this mode, OpenShift needs to start its in-cluster
CoreDNS based DNS solution for API, API-Int and Ingress resolution
so that cluster installation is successful. The customer is
expected to configure their DNS solution post-install.

At this the time, the cloud's API and API-Int LBs are configured
by the Installer and its value is not expected to change during
the life of the cluster. The Ingress operator continues to handle
Ingress LB configuration.

Based on enhancement: openshift/enhancements#1468

cmd/corednsmonitor/corednsmonitor.go

@@ -55,8 +55,20 @@ func main() {
 			if err != nil {
 				return err
 			}
+			cloudExtLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+			if err != nil {
+				cloudExtLBIPs = []net.IP{}
+			}
+			cloudIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+			if err != nil {
+				cloudIntLBIPs = []net.IP{}
+			}
+			cloudIngressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips")
+			if err != nil {
+				cloudIngressLBIPs = []net.IP{}
+			}
 
-			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval)
+			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval, cloudExtLBIPs, cloudIntLBIPs, cloudIngressLBIPs)
 		},
 	}
 	rootCmd.PersistentFlags().StringP("cluster-config", "c", "", "Path to cluster-config ConfigMap to retrieve ControlPlane info")

cmd/runtimecfg/display.go

@@ -28,6 +28,9 @@ func init() {
 	displayCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	displayCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	displayCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	displayCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	displayCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift API")
+	displayCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers")
 	rootCmd.AddCommand(displayCmd)
 }
 
@@ -84,7 +87,22 @@ func runDisplay(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips")
+	if err != nil {
+		ingressLBIPs = []net.IP{}
+	}
+	clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs}
+
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig)
 	if err != nil {
 		return err
 	}

cmd/runtimecfg/render.go

@@ -33,6 +33,9 @@ func init() {
 	renderCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	renderCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	renderCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	renderCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	renderCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift Internal API")
+	renderCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers")
 	rootCmd.AddCommand(renderCmd)
 }
 
@@ -88,7 +91,21 @@ func runRender(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips")
+	if err != nil {
+		ingressLBIPs = []net.IP{}
+	}
+	clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs}
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig)
 	if err != nil {
 		return err
 	}

pkg/config/cloud_lb_config_test.go

@@ -0,0 +1,45 @@
+package config
+
+import (
+	"net"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	testKubeconfigPath    = "/test/path/kubeconfig"
+	testClusterConfigPath = "/test/path/clusterConfig"
+	testResolvConfPath    = "/test/path/resolvConf"
+	testApiLBIPv4         = net.ParseIP("192.168.0.111")
+	testApiIntLBIPv4      = net.ParseIP("10.10.10.20")
+	testIngressOneIPv4    = net.ParseIP("192.168.20.140")
+	testIngressTwoIPv4    = net.ParseIP("10.10.10.40")
+	testClusterLBConfig   = ClusterLBConfig{
+		ApiLBIPs:     []net.IP{testApiLBIPv4},
+		ApiIntLBIPs:  []net.IP{testApiIntLBIPv4},
+		IngressLBIPs: []net.IP{testIngressOneIPv4, testIngressTwoIPv4}}
+	expectedApiLBIPv4      = "192.168.0.111"
+	expectedIngressTwoIPv4 = "10.10.10.40"
+	testNode               = Node{}
+)
+
+var _ = Describe("PopulateCloudLBIPAddresses", func() {
+	Context("for IPV4 Cloud LB IPs", func() {
+		Context("with multiple Ingress LB IPs", func() {
+			It("matches multiple IPs in 1 node", func() {
+				testNode, err := PopulateCloudLBIPAddresses(testClusterLBConfig, testNode)
+				Expect(testNode.Cluster.APILBIPs[0]).To(Equal(expectedApiLBIPv4))
+				Expect(testNode.Cluster.IngressLBIPs[1]).To(Equal(expectedIngressTwoIPv4))
+				Expect(testNode.Cluster.CloudLBRecordType).To(Equal("A"))
+				Expect(err).To(BeNil())
+			})
+		})
+	})
+})
+
+func TestCloudLBConfig(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Cloud LB Config Tests")
+}

pkg/config/node.go

@@ -52,6 +52,11 @@ type Cluster struct {
 	VIPNetmask             int
 	MasterAmount           int64
 	NodeAddresses          []NodeAddress
+	APILBIPs               []string
+	APIIntLBIPs            []string
+	IngressLBIPs           []string
+	CloudLBRecordType      string
+	CloudLBEmptyType       string
 }
 
 type Backend struct {
@@ -84,6 +89,12 @@ type Node struct {
 	Configs       *[]Node
 }
 
+type ClusterLBConfig struct {
+	ApiLBIPs     []net.IP
+	ApiIntLBIPs  []net.IP
+	IngressLBIPs []net.IP
+}
+
 func getDNSUpstreams(resolvConfPath string) (upstreams []string, err error) {
 	dnsFile, err := os.Open(resolvConfPath)
 	if err != nil {
@@ -417,34 +428,42 @@ func getNodeIpForRequestedIpStack(node v1.Node, filterIps []string, machineNetwo
 // apiPort: The port on which the k8s api listens. Should be 6443.
 // lbPort: The port on which haproxy listens.
 // statPort: The port on which the haproxy stats endpoint listens.
-func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
-	vipCount := 0
-	if len(apiVips) > len(ingressVips) {
-		vipCount = len(apiVips)
-	} else {
-		vipCount = len(ingressVips)
-	}
-	nodes := []Node{}
-	var apiVip, ingressVip net.IP
-	for i := 0; i < vipCount; i++ {
-		if i < len(apiVips) {
-			apiVip = apiVips[i]
-		} else {
-			apiVip = nil
-		}
-		if i < len(ingressVips) {
-			ingressVip = ingressVips[i]
+// clusterLBConfig: A struct containing IPs for API, API-Int and Ingress LBs
+func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16, clusterLBConfig ClusterLBConfig) (node Node, err error) {
+	switch {
+	case len(clusterLBConfig.ApiIntLBIPs) > 0:
+		// Cloud Platforms with cloud LBs but no Cloud DNS
+		return getConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath, clusterLBConfig)
+	default:
+		// On-prem platforms
+		vipCount := 0
+		if len(apiVips) > len(ingressVips) {
+			vipCount = len(apiVips)
 		} else {
-			ingressVip = nil
+			vipCount = len(ingressVips)
 		}
-		newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort)
-		if err != nil {
-			return Node{}, err
+		nodes := []Node{}
+		var apiVip, ingressVip net.IP
+		for i := 0; i < vipCount; i++ {
+			if i < len(apiVips) {
+				apiVip = apiVips[i]
+			} else {
+				apiVip = nil
+			}
+			if i < len(ingressVips) {
+				ingressVip = ingressVips[i]
+			} else {
+				ingressVip = nil
+			}
+			newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort)
+			if err != nil {
+				return Node{}, err
+			}
+			nodes = append(nodes, newNode)
 		}
-		nodes = append(nodes, newNode)
+		nodes[0].Configs = &nodes
+		return nodes[0], nil
 	}
-	nodes[0].Configs = &nodes
-	return nodes[0], nil
 }
 
 func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
@@ -694,3 +713,80 @@ func PopulateNodeAddresses(kubeconfigPath string, node *Node) {
 		}
 	}
 }
+
+func getConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath string, clusterLBConfig ClusterLBConfig) (node Node, err error) {
+	var apiLBIP, apiIntLBIP, ingressIP net.IP
+	nodes := []Node{}
+	ipCount := 0
+	if len(clusterLBConfig.ApiIntLBIPs) > len(clusterLBConfig.IngressLBIPs) {
+		ipCount = len(clusterLBConfig.ApiIntLBIPs)
+	} else {
+		ipCount = len(clusterLBConfig.IngressLBIPs)
+	}
+	for i := 0; i < ipCount; i++ {
+		if i < len(clusterLBConfig.ApiIntLBIPs) {
+			apiIntLBIP = clusterLBConfig.ApiIntLBIPs[i]
+		} else {
+			apiIntLBIP = nil
+		}
+
+		// For public clusters. Private clusters will not have External
+		// LBs so apiLBIPs could be empty.
+		if len(clusterLBConfig.ApiLBIPs) != 0 && i < len(clusterLBConfig.ApiLBIPs) {
+			apiLBIP = clusterLBConfig.ApiLBIPs[i]
+		} else {
+			apiLBIP = nil
+		}
+		if len(clusterLBConfig.IngressLBIPs) != 0 && i < len(clusterLBConfig.IngressLBIPs) {
+			ingressIP = clusterLBConfig.IngressLBIPs[i]
+		} else {
+			ingressIP = nil
+		}
+		newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, nil, nil, 0, 0, 0)
+		if err != nil {
+			return Node{}, err
+		}
+		newNode = updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP, newNode)
+		nodes = append(nodes, newNode)
+	}
+	nodes[0].Configs = &nodes
+	return nodes[0], nil
+}
+
+func updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP net.IP, node Node) Node {
+	if apiIntLBIP != nil {
+		node.Cluster.CloudLBRecordType = "A"
+		node.Cluster.CloudLBEmptyType = "AAAA"
+		if apiIntLBIP.To4() == nil {
+			node.Cluster.CloudLBRecordType = "AAAA"
+			node.Cluster.CloudLBEmptyType = "A"
+		}
+		node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIP.String())
+	}
+	if apiLBIP != nil {
+		node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIP.String())
+	}
+	if ingressIP != nil {
+		node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ingressIP.String())
+	}
+	return node
+}
+
+func PopulateCloudLBIPAddresses(clusterLBConfig ClusterLBConfig, node Node) (updatedNode Node, err error) {
+	for _, ip := range clusterLBConfig.ApiIntLBIPs {
+		node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, ip.String())
+	}
+	for _, ip := range clusterLBConfig.ApiLBIPs {
+		node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, ip.String())
+	}
+	for _, ip := range clusterLBConfig.IngressLBIPs {
+		node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ip.String())
+	}
+	node.Cluster.CloudLBRecordType = "A"
+	node.Cluster.CloudLBEmptyType = "AAAA"
+	if len(clusterLBConfig.ApiIntLBIPs) > 0 && clusterLBConfig.ApiIntLBIPs[0].To4() == nil {
+		node.Cluster.CloudLBRecordType = "AAAA"
+		node.Cluster.CloudLBEmptyType = "A"
+	}
+	return node, nil
+}

pkg/monitor/corednsmonitor.go

@@ -16,7 +16,7 @@ import (
 
 const resolvConfFilepath string = "/var/run/NetworkManager/resolv.conf"
 
-func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration) error {
+func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration, apiLBIPs, apiIntLBIPs, ingressLBIPs []net.IP) error {
 	signals := make(chan os.Signal, 1)
 	done := make(chan bool, 1)
 
@@ -42,10 +42,15 @@ func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath strin
 			if err != nil {
 				return err
 			}
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0)
+			clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs}
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0, clusterLBConfig)
 			if err != nil {
 				return err
 			}
+			// Populate cloud LB IP addresses for platforms where the cloud LBs
+			// have already been configured
+			newConfig, _ = config.PopulateCloudLBIPAddresses(clusterLBConfig, newConfig)
+
 			config.PopulateNodeAddresses(kubeconfigPath, &newConfig)
 			// There should never be 0 nodes in a functioning cluster. This means
 			// we failed to populate the list, so we don't want to render.

pkg/monitor/dnsmasqmonitor.go

@@ -33,7 +33,7 @@ func DnsmasqWatch(kubeconfigPath, templatePath, cfgPath string, apiVips []net.IP
 			return nil
 		default:
 			// We only care about the api vip and cluster domain here
-			config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0)
+			config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0, config.ClusterLBConfig{})
 			if err != nil {
 				return err
 			}

pkg/monitor/dynkeepalived.go

@@ -355,7 +355,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st
 
 		case desiredModeInfo := <-updateModeCh:
 
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, config.ClusterLBConfig{})
 			if err != nil {
 				return err
 			}
@@ -437,7 +437,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st
 					}
 				}
 			}
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, config.ClusterLBConfig{})
 			if err != nil {
 				return err
 			}