CORS-2818: Adding ability to render Cloud LB IPs

cmd/corednsmonitor/corednsmonitor.go

@@ -55,8 +55,20 @@ func main() {
 			if err != nil {
 				return err
 			}
+			cloudExtLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+			if err != nil {
+				cloudExtLBIPs = []net.IP{}
+			}
+			cloudIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+			if err != nil {
+				cloudIntLBIPs = []net.IP{}
+			}
+			cloudIngressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips")
+			if err != nil {
+				cloudIngressLBIPs = []net.IP{}
+			}
 
-			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval)
+			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval, cloudExtLBIPs, cloudIntLBIPs, cloudIngressLBIPs)
 		},
 	}
 	rootCmd.PersistentFlags().StringP("cluster-config", "c", "", "Path to cluster-config ConfigMap to retrieve ControlPlane info")

cmd/runtimecfg/display.go

@@ -28,6 +28,9 @@ func init() {
 	displayCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	displayCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	displayCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	displayCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	displayCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift API")
+	displayCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers")
 	rootCmd.AddCommand(displayCmd)
 }
 
@@ -84,7 +87,22 @@ func runDisplay(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips")
+	if err != nil {
+		ingressLBIPs = []net.IP{}
+	}
+	clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs}
+
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig)
 	if err != nil {
 		return err
 	}

cmd/runtimecfg/render.go

@@ -33,6 +33,9 @@ func init() {
 	renderCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	renderCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	renderCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	renderCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	renderCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift Internal API")
+	renderCmd.Flags().IPSlice("cloud-ingress-lb-ips", nil, "IP Addresses of Cloud Ingress Load Balancers")
 	rootCmd.AddCommand(renderCmd)
 }
 
@@ -88,7 +91,21 @@ func runRender(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	ingressLBIPs, err := cmd.Flags().GetIPSlice("cloud-ingress-lb-ips")
+	if err != nil {
+		ingressLBIPs = []net.IP{}
+	}
+	clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs}
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, clusterLBConfig)
 	if err != nil {
 		return err
 	}

pkg/config/node.go

@@ -52,6 +52,11 @@ type Cluster struct {
 	VIPNetmask             int
 	MasterAmount           int64
 	NodeAddresses          []NodeAddress
+	APILBIPs               []string
+	APIIntLBIPs            []string
+	IngressLBIPs           []string
+	CloudLBRecordType      string
+	CloudLBEmptyType       string
 }
 
 type Backend struct {
@@ -84,6 +89,12 @@ type Node struct {
 	Configs       *[]Node
 }
 
+type ClusterLBConfig struct {
+	ApiLBIPs     []net.IP
+	ApiIntLBIPs  []net.IP
+	IngressLBIPs []net.IP
+}
+
 func getDNSUpstreams(resolvConfPath string) (upstreams []string, err error) {
 	dnsFile, err := os.Open(resolvConfPath)
 	if err != nil {
@@ -417,7 +428,13 @@ func getNodeIpForRequestedIpStack(node v1.Node, filterIps []string, machineNetwo
 // apiPort: The port on which the k8s api listens. Should be 6443.
 // lbPort: The port on which haproxy listens.
 // statPort: The port on which the haproxy stats endpoint listens.
-func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
+// clusterLBConfig: A struct containing IPs for API, API-Int and Ingress LBs
+func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16, clusterLBConfig ClusterLBConfig) (node Node, err error) {
+	if len(clusterLBConfig.ApiIntLBIPs) > 0 {
+		// Cloud Platforms with cloud LBs but no Cloud DNS
+		return getNodeConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath, clusterLBConfig)
+	}
+	// On-prem platforms
 	vipCount := 0
 	if len(apiVips) > len(ingressVips) {
 		vipCount = len(apiVips)
@@ -694,3 +711,94 @@ func PopulateNodeAddresses(kubeconfigPath string, node *Node) {
 		}
 	}
 }
+
+func getNodeConfigWithCloudLBIPs(kubeconfigPath, clusterConfigPath, resolvConfPath string, clusterLBConfig ClusterLBConfig) (node Node, err error) {
+	var apiLBIP, apiIntLBIP, ingressIP net.IP
+	nodes := []Node{}
+
+	// The number of LB IPs provided in ClusterLBConfig's ApiIntLBIPs, ApiLBIPs and IngressLBIPs
+	// could all be different. In private clusters, there would be no API LB IPs provided. So,to
+	// account for the varying lengths we determine the longest between the list of Ingress LB
+	// IPs and API-Int LB IPs. When present, len(ApiLBIPs) is equal to len(ApiIntLBIPs).
+	ipCount := 0
+	if len(clusterLBConfig.ApiIntLBIPs) > len(clusterLBConfig.IngressLBIPs) {
+		ipCount = len(clusterLBConfig.ApiIntLBIPs)
+	} else {
+		ipCount = len(clusterLBConfig.IngressLBIPs)
+	}
+
+	// Iterate through the longest list of LB IPs and provide an API, API-Int and Ingress
+	// LB IP to each newly created Node. When a list has no more LB IPs, update Node object
+	// with a nil IP address.
+	for i := 0; i < ipCount; i++ {
+		if i < len(clusterLBConfig.ApiIntLBIPs) {
+			apiIntLBIP = clusterLBConfig.ApiIntLBIPs[i]
+		} else {
+			apiIntLBIP = nil
+		}
+
+		// For public clusters. Private clusters will not have External
+		// LBs so apiLBIPs could be empty.
+		if len(clusterLBConfig.ApiLBIPs) != 0 && i < len(clusterLBConfig.ApiLBIPs) {
+			apiLBIP = clusterLBConfig.ApiLBIPs[i]
+		} else {
+			apiLBIP = nil
+		}
+		if len(clusterLBConfig.IngressLBIPs) != 0 && i < len(clusterLBConfig.IngressLBIPs) {
+			ingressIP = clusterLBConfig.IngressLBIPs[i]
+		} else {
+			ingressIP = nil
+		}
+		newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, nil, nil, 0, 0, 0)
+		if err != nil {
+			return Node{}, err
+		}
+		newNode = updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP, newNode)
+		nodes = append(nodes, newNode)
+	}
+	nodes[0].Configs = &nodes
+	return nodes[0], nil
+}
+
+func updateNodewithCloudLBIPs(apiLBIP, apiIntLBIP, ingressIP net.IP, node Node) Node {
+	var validLBIP net.IP
+	if apiIntLBIP != nil {
+		validLBIP = apiIntLBIP
+		node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIP.String())
+	}
+	if apiLBIP != nil {
+		node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIP.String())
+	}
+	if ingressIP != nil {
+		node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ingressIP.String())
+		if validLBIP == nil {
+			validLBIP = ingressIP
+		}
+	}
+	node.Cluster.CloudLBRecordType = "A"
+	node.Cluster.CloudLBEmptyType = "AAAA"
+	if validLBIP.To4() == nil {
+		node.Cluster.CloudLBRecordType = "AAAA"
+		node.Cluster.CloudLBEmptyType = "A"
+	}
+	return node
+}
+
+func PopulateCloudLBIPAddresses(clusterLBConfig ClusterLBConfig, node Node) (updatedNode Node, err error) {
+	for _, ip := range clusterLBConfig.ApiIntLBIPs {
+		node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, ip.String())
+	}
+	for _, ip := range clusterLBConfig.ApiLBIPs {
+		node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, ip.String())
+	}
+	for _, ip := range clusterLBConfig.IngressLBIPs {
+		node.Cluster.IngressLBIPs = append(node.Cluster.IngressLBIPs, ip.String())
+	}
+	node.Cluster.CloudLBRecordType = "A"
+	node.Cluster.CloudLBEmptyType = "AAAA"
+	if len(clusterLBConfig.ApiIntLBIPs) > 0 && clusterLBConfig.ApiIntLBIPs[0].To4() == nil {
+		node.Cluster.CloudLBRecordType = "AAAA"
+		node.Cluster.CloudLBEmptyType = "A"
+	}
+	return node, nil
+}

pkg/config/node_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"net"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -140,6 +141,136 @@ var _ = Describe("getNodePeersForIpStack", func() {
 	})
 })
 
+// Following needed for cloud LB IP tests
+var (
+	testKubeconfigPath    = "/test/path/kubeconfig"
+	testClusterConfigPath = "/test/path/clusterConfig"
+	testResolvConfPath    = "/test/path/resolvConf"
+	testApiLBIPv4         = net.ParseIP("192.168.0.111")
+	testApiIntLBIPv4      = net.ParseIP("10.10.10.20")
+	testIngressOneIPv4    = net.ParseIP("192.168.20.140")
+	testIngressTwoIPv4    = net.ParseIP("10.10.10.40")
+	testClusterLBConfig   = ClusterLBConfig{
+		ApiLBIPs:     []net.IP{testApiLBIPv4},
+		ApiIntLBIPs:  []net.IP{testApiIntLBIPv4},
+		IngressLBIPs: []net.IP{testIngressOneIPv4, testIngressTwoIPv4}}
+	expectedApiLBIPv4      = "192.168.0.111"
+	expectedApiIntLBIPv4   = "10.10.10.20"
+	expectedIngressOneIPv4 = "192.168.20.140"
+	expectedIngressTwoIPv4 = "10.10.10.40"
+
+	emptyLBIPs = []net.IP{}
+)
+
+var _ = Describe("PopulateCloudLBIPAddresses", func() {
+	Context("for IPV4 Cloud LB IPs", func() {
+		Context("with multiple Ingress LB IPs", func() {
+			It("matches IPv4 API and Ingress LB IPs", func() {
+				newNode := Node{}
+				newNode, err := PopulateCloudLBIPAddresses(testClusterLBConfig, newNode)
+				Expect(newNode.Cluster.APILBIPs[0]).To(Equal(expectedApiLBIPv4))
+				Expect(newNode.Cluster.IngressLBIPs[1]).To(Equal(expectedIngressTwoIPv4))
+				Expect(newNode.Cluster.CloudLBRecordType).To(Equal("A"))
+				Expect(err).To(BeNil())
+			})
+			It("handles Empty API LB IPs", func() {
+				newNode := Node{}
+				// Empty API LB IP
+				emptyApiLBIPLBConfig := ClusterLBConfig{
+					ApiLBIPs:     []net.IP{},
+					ApiIntLBIPs:  []net.IP{testApiIntLBIPv4},
+					IngressLBIPs: []net.IP{testIngressOneIPv4}}
+				newNode, err := PopulateCloudLBIPAddresses(emptyApiLBIPLBConfig, newNode)
+				Expect(len(newNode.Cluster.APILBIPs)).To(Equal(len(emptyLBIPs)))
+				Expect(newNode.Cluster.APIIntLBIPs[0]).To(Equal(expectedApiIntLBIPv4))
+				Expect(newNode.Cluster.IngressLBIPs[0]).To(Equal(expectedIngressOneIPv4))
+				Expect(newNode.Cluster.CloudLBRecordType).To(Equal("A"))
+				Expect(err).To(BeNil())
+			})
+			It("handles Empty API Int LB IPs", func() {
+				newNode := Node{}
+				// Empty API-Int LB IP
+				emptyApiIntLBIPLBConfig := ClusterLBConfig{
+					ApiLBIPs:     []net.IP{testApiLBIPv4},
+					ApiIntLBIPs:  []net.IP{},
+					IngressLBIPs: []net.IP{testIngressOneIPv4}}
+				newNode, err := PopulateCloudLBIPAddresses(emptyApiIntLBIPLBConfig, newNode)
+				Expect(newNode.Cluster.APILBIPs[0]).To(Equal(expectedApiLBIPv4))
+				Expect(len(newNode.Cluster.APIIntLBIPs)).To(Equal(len(emptyLBIPs)))
+				Expect(newNode.Cluster.IngressLBIPs[0]).To(Equal(expectedIngressOneIPv4))
+				Expect(newNode.Cluster.CloudLBRecordType).To(Equal("A"))
+				Expect(err).To(BeNil())
+			})
+			It("handles Empty Ingress LB IPs", func() {
+				newNode := Node{}
+				// Empty Ingress LB IP
+				emptyIngressLBIPLBConfig := ClusterLBConfig{
+					ApiLBIPs:     []net.IP{testApiLBIPv4},
+					ApiIntLBIPs:  []net.IP{testApiIntLBIPv4},
+					IngressLBIPs: []net.IP{}}
+				newNode, err := PopulateCloudLBIPAddresses(emptyIngressLBIPLBConfig, newNode)
+				Expect(newNode.Cluster.APILBIPs[0]).To(Equal(expectedApiLBIPv4))
+				Expect(newNode.Cluster.APIIntLBIPs[0]).To(Equal(expectedApiIntLBIPv4))
+				Expect(len(newNode.Cluster.IngressLBIPs)).To(Equal(len(emptyLBIPs)))
+				Expect(newNode.Cluster.CloudLBRecordType).To(Equal("A"))
+				Expect(err).To(BeNil())
+			})
+			It("handles Empty All LB IPs", func() {
+				newNode := Node{}
+				// Empty All LB IPs
+				emptyAllLBIPLBConfig := ClusterLBConfig{
+					ApiLBIPs:     []net.IP{},
+					ApiIntLBIPs:  []net.IP{},
+					IngressLBIPs: []net.IP{}}
+				newNode, err := PopulateCloudLBIPAddresses(emptyAllLBIPLBConfig, newNode)
+				Expect(len(newNode.Cluster.APILBIPs)).To(Equal(len(emptyLBIPs)))
+				Expect(len(newNode.Cluster.APIIntLBIPs)).To(Equal(len(emptyLBIPs)))
+				Expect(len(newNode.Cluster.IngressLBIPs)).To(Equal(len(emptyLBIPs)))
+				Expect(newNode.Cluster.CloudLBRecordType).To(Equal("A"))
+				Expect(err).To(BeNil())
+			})
+		})
+	})
+})
+
+var _ = Describe("updateNodewithCloudLBIPs", func() {
+	Context("for IPV4 Cloud LB IPs", func() {
+		Context("with one LB IP per Node", func() {
+			It("matches IPv4 API and Ingress LB IPs", func() {
+				updateNode := Node{}
+				updateNode = updateNodewithCloudLBIPs(testApiLBIPv4, testApiIntLBIPv4, testIngressOneIPv4, updateNode)
+				Expect(updateNode.Cluster.APIIntLBIPs[0]).To(Equal(expectedApiIntLBIPv4))
+				Expect(updateNode.Cluster.IngressLBIPs[0]).To(Equal(expectedIngressOneIPv4))
+				Expect(updateNode.Cluster.CloudLBRecordType).To(Equal("A"))
+			})
+			It("handles nil API LB IP", func() {
+				updateNode := Node{}
+				updateNode = updateNodewithCloudLBIPs(nil, testApiIntLBIPv4, testIngressOneIPv4, updateNode)
+				Expect(len(updateNode.Cluster.APILBIPs)).To(Equal(0))
+				Expect(updateNode.Cluster.APIIntLBIPs[0]).To(Equal(expectedApiIntLBIPv4))
+				Expect(updateNode.Cluster.IngressLBIPs[0]).To(Equal(expectedIngressOneIPv4))
+				Expect(updateNode.Cluster.CloudLBRecordType).To(Equal("A"))
+			})
+			It("handles nil API-Int LB IP", func() {
+				updateNode := Node{}
+				updateNode = updateNodewithCloudLBIPs(testApiLBIPv4, nil, testIngressOneIPv4, updateNode)
+				Expect(updateNode.Cluster.APILBIPs[0]).To(Equal(expectedApiLBIPv4))
+				Expect(len(updateNode.Cluster.APIIntLBIPs)).To(Equal(0))
+				Expect(updateNode.Cluster.IngressLBIPs[0]).To(Equal(expectedIngressOneIPv4))
+				Expect(updateNode.Cluster.CloudLBRecordType).To(Equal("A"))
+			})
+			It("handles nil API and Ingress LBs IP", func() {
+				updateNode := Node{}
+				updateNode = updateNodewithCloudLBIPs(nil, testApiIntLBIPv4, nil, updateNode)
+				Expect(updateNode.Cluster.APIIntLBIPs[0]).To(Equal(expectedApiIntLBIPv4))
+				Expect(len(updateNode.Cluster.APILBIPs)).To(Equal(0))
+				Expect(len(updateNode.Cluster.IngressLBIPs)).To(Equal(0))
+				Expect(updateNode.Cluster.CloudLBRecordType).To(Equal("A"))
+			})
+		})
+	})
+})
+
 func Test(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Config tests")

pkg/monitor/corednsmonitor.go

@@ -16,7 +16,7 @@ import (
 
 const resolvConfFilepath string = "/var/run/NetworkManager/resolv.conf"
 
-func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration) error {
+func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration, apiLBIPs, apiIntLBIPs, ingressLBIPs []net.IP) error {
 	signals := make(chan os.Signal, 1)
 	done := make(chan bool, 1)
 
@@ -42,10 +42,19 @@ func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath strin
 			if err != nil {
 				return err
 			}
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0)
+			clusterLBConfig := config.ClusterLBConfig{ApiLBIPs: apiLBIPs, ApiIntLBIPs: apiIntLBIPs, IngressLBIPs: ingressLBIPs}
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0, clusterLBConfig)
 			if err != nil {
 				return err
 			}
+
+			// Populate cloud LB IP addresses for platforms where the cloud LBs
+			// have already been configured
+			newConfig, err = config.PopulateCloudLBIPAddresses(clusterLBConfig, newConfig)
+			if err != nil {
+				return err
+			}
+
 			config.PopulateNodeAddresses(kubeconfigPath, &newConfig)
 			// There should never be 0 nodes in a functioning cluster. This means
 			// we failed to populate the list, so we don't want to render.